2024-12-04 14:58:19,836 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-04 14:58:19,850 main DEBUG Took 0.011873 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-04 14:58:19,851 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-04 14:58:19,851 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-04 14:58:19,852 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-04 14:58:19,853 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:58:19,861 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-04 14:58:19,873 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:58:19,874 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:58:19,875 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:58:19,875 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:58:19,875 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:58:19,876 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:58:19,876 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:58:19,877 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:58:19,877 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:58:19,877 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:58:19,878 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:58:19,878 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:58:19,879 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:58:19,879 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:58:19,879 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:58:19,879 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:58:19,880 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:58:19,880 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:58:19,880 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:58:19,881 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:58:19,881 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:58:19,881 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:58:19,882 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:58:19,882 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:58:19,882 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:58:19,883 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-04 14:58:19,884 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:58:19,885 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-04 14:58:19,887 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-04 14:58:19,887 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-04 14:58:19,888 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-04 14:58:19,889 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-04 14:58:19,896 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-04 14:58:19,898 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-04 14:58:19,900 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-04 14:58:19,900 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-04 14:58:19,901 main DEBUG createAppenders(={Console}) 2024-12-04 14:58:19,901 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-04 14:58:19,902 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-04 14:58:19,902 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-04 14:58:19,902 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-04 14:58:19,903 main DEBUG OutputStream closed 2024-12-04 14:58:19,903 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-04 14:58:19,903 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-04 14:58:19,903 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-04 14:58:19,964 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-04 14:58:19,966 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-04 14:58:19,967 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-04 14:58:19,968 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-04 14:58:19,969 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-04 14:58:19,969 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-04 14:58:19,969 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-04 14:58:19,969 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-04 14:58:19,970 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-04 14:58:19,970 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-04 14:58:19,970 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-04 14:58:19,970 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-04 14:58:19,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-04 14:58:19,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-04 14:58:19,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-04 14:58:19,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-04 14:58:19,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-04 14:58:19,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-04 14:58:19,974 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04 14:58:19,974 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-04 14:58:19,975 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-04 14:58:19,975 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-04T14:58:20,168 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b 2024-12-04 14:58:20,171 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-04 14:58:20,171 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04T14:58:20,179 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-04T14:58:20,195 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T14:58:20,209 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/cluster_5448908e-37df-abd5-da05-dbe928ab313c, deleteOnExit=true 2024-12-04T14:58:20,209 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T14:58:20,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/test.cache.data in system properties and HBase conf 2024-12-04T14:58:20,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T14:58:20,211 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/hadoop.log.dir in system properties and HBase conf 2024-12-04T14:58:20,211 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T14:58:20,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T14:58:20,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T14:58:20,309 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-04T14:58:20,389 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T14:58:20,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T14:58:20,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T14:58:20,393 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T14:58:20,393 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T14:58:20,394 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T14:58:20,394 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T14:58:20,394 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T14:58:20,395 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T14:58:20,395 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T14:58:20,395 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/nfs.dump.dir in system properties and HBase conf 2024-12-04T14:58:20,396 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/java.io.tmpdir in system properties and HBase conf 2024-12-04T14:58:20,396 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T14:58:20,396 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T14:58:20,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T14:58:21,245 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-04T14:58:21,315 INFO [Time-limited test {}] log.Log(170): Logging initialized @2128ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-04T14:58:21,389 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:58:21,444 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:58:21,462 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:58:21,462 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:58:21,463 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T14:58:21,475 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:58:21,477 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:58:21,478 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:58:21,641 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2341a969{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/java.io.tmpdir/jetty-localhost-45349-hadoop-hdfs-3_4_1-tests_jar-_-any-4725692402879234301/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T14:58:21,651 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:45349} 2024-12-04T14:58:21,652 INFO [Time-limited test {}] server.Server(415): Started @2466ms 2024-12-04T14:58:22,116 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:58:22,123 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:58:22,124 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:58:22,124 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:58:22,124 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T14:58:22,125 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:58:22,125 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:58:22,219 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@292d50c4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/java.io.tmpdir/jetty-localhost-35885-hadoop-hdfs-3_4_1-tests_jar-_-any-1093842331080498193/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:58:22,220 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:35885} 2024-12-04T14:58:22,220 INFO [Time-limited test {}] server.Server(415): Started @3034ms 2024-12-04T14:58:22,265 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:58:23,072 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/cluster_5448908e-37df-abd5-da05-dbe928ab313c/data/data2/current/BP-1917135717-172.17.0.2-1733324300869/current, will proceed with Du for space computation calculation, 2024-12-04T14:58:23,072 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/cluster_5448908e-37df-abd5-da05-dbe928ab313c/data/data1/current/BP-1917135717-172.17.0.2-1733324300869/current, will proceed with Du for space computation calculation, 2024-12-04T14:58:23,102 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:58:23,143 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcb9add9f5adfd194 with lease ID 0xe677634ff111389c: Processing first storage report for DS-40b04e37-5102-422a-9b3a-9b86db1d47b1 from datanode DatanodeRegistration(127.0.0.1:44941, datanodeUuid=dc1a60eb-c1c2-4629-8cb3-41f5de55f15a, infoPort=41829, infoSecurePort=0, ipcPort=39287, storageInfo=lv=-57;cid=testClusterID;nsid=1801599307;c=1733324300869) 2024-12-04T14:58:23,144 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcb9add9f5adfd194 with lease ID 0xe677634ff111389c: from storage DS-40b04e37-5102-422a-9b3a-9b86db1d47b1 node DatanodeRegistration(127.0.0.1:44941, datanodeUuid=dc1a60eb-c1c2-4629-8cb3-41f5de55f15a, infoPort=41829, infoSecurePort=0, ipcPort=39287, storageInfo=lv=-57;cid=testClusterID;nsid=1801599307;c=1733324300869), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T14:58:23,145 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcb9add9f5adfd194 with lease ID 0xe677634ff111389c: Processing first storage report for DS-ba702a78-1d36-45c7-a4e3-0f7d292129a9 from datanode DatanodeRegistration(127.0.0.1:44941, datanodeUuid=dc1a60eb-c1c2-4629-8cb3-41f5de55f15a, infoPort=41829, infoSecurePort=0, ipcPort=39287, storageInfo=lv=-57;cid=testClusterID;nsid=1801599307;c=1733324300869) 2024-12-04T14:58:23,145 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcb9add9f5adfd194 with lease ID 0xe677634ff111389c: from storage DS-ba702a78-1d36-45c7-a4e3-0f7d292129a9 node DatanodeRegistration(127.0.0.1:44941, datanodeUuid=dc1a60eb-c1c2-4629-8cb3-41f5de55f15a, infoPort=41829, infoSecurePort=0, ipcPort=39287, storageInfo=lv=-57;cid=testClusterID;nsid=1801599307;c=1733324300869), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:58:23,229 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b 2024-12-04T14:58:23,339 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/cluster_5448908e-37df-abd5-da05-dbe928ab313c/zookeeper_0, clientPort=52559, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/cluster_5448908e-37df-abd5-da05-dbe928ab313c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/cluster_5448908e-37df-abd5-da05-dbe928ab313c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T14:58:23,347 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52559 2024-12-04T14:58:23,356 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:58:23,359 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:58:23,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741825_1001 (size=7) 2024-12-04T14:58:23,962 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee with version=8 2024-12-04T14:58:23,962 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/hbase-staging 2024-12-04T14:58:24,038 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-04T14:58:24,261 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d214edae7bd2:0 server-side Connection retries=45 2024-12-04T14:58:24,269 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:58:24,269 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T14:58:24,273 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T14:58:24,273 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:58:24,274 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T14:58:24,391 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T14:58:24,441 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-04T14:58:24,449 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-04T14:58:24,453 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T14:58:24,475 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 108950 (auto-detected) 2024-12-04T14:58:24,476 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-04T14:58:24,493 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39449 2024-12-04T14:58:24,513 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39449 connecting to ZooKeeper ensemble=127.0.0.1:52559 2024-12-04T14:58:24,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:394490x0, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T14:58:24,617 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39449-0x101a2238e4c0000 connected 2024-12-04T14:58:24,695 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:58:24,699 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:58:24,710 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:58:24,714 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee, hbase.cluster.distributed=false 2024-12-04T14:58:24,734 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T14:58:24,738 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39449 2024-12-04T14:58:24,739 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39449 2024-12-04T14:58:24,739 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39449 2024-12-04T14:58:24,739 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39449 2024-12-04T14:58:24,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39449 2024-12-04T14:58:24,832 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d214edae7bd2:0 server-side Connection retries=45 2024-12-04T14:58:24,833 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:58:24,834 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T14:58:24,834 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T14:58:24,834 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:58:24,834 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T14:58:24,837 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T14:58:24,840 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T14:58:24,841 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36649 2024-12-04T14:58:24,842 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36649 connecting to ZooKeeper ensemble=127.0.0.1:52559 2024-12-04T14:58:24,843 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:58:24,846 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:58:24,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:366490x0, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T14:58:24,860 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:366490x0, quorum=127.0.0.1:52559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:58:24,860 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36649-0x101a2238e4c0001 connected 2024-12-04T14:58:24,863 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T14:58:24,869 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T14:58:24,872 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36649-0x101a2238e4c0001, quorum=127.0.0.1:52559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T14:58:24,877 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36649-0x101a2238e4c0001, quorum=127.0.0.1:52559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T14:58:24,878 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36649 2024-12-04T14:58:24,878 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36649 2024-12-04T14:58:24,878 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36649 2024-12-04T14:58:24,880 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36649 2024-12-04T14:58:24,880 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36649 2024-12-04T14:58:24,893 DEBUG [M:0;d214edae7bd2:39449 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d214edae7bd2:39449 2024-12-04T14:58:24,894 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d214edae7bd2,39449,1733324304115 2024-12-04T14:58:24,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:58:24,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36649-0x101a2238e4c0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:58:24,910 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d214edae7bd2,39449,1733324304115 2024-12-04T14:58:24,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36649-0x101a2238e4c0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T14:58:24,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:58:24,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36649-0x101a2238e4c0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:58:24,942 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T14:58:24,943 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d214edae7bd2,39449,1733324304115 from backup master directory 2024-12-04T14:58:24,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d214edae7bd2,39449,1733324304115 2024-12-04T14:58:24,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36649-0x101a2238e4c0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:58:24,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:58:24,951 WARN [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T14:58:24,951 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d214edae7bd2,39449,1733324304115 2024-12-04T14:58:24,954 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-04T14:58:24,955 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-04T14:58:25,005 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/hbase.id] with ID: e66e92f8-1d08-41ec-82b1-ea6168d3b6c4 2024-12-04T14:58:25,005 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/.tmp/hbase.id 2024-12-04T14:58:25,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741826_1002 (size=42) 2024-12-04T14:58:25,420 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/.tmp/hbase.id]:[hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/hbase.id] 2024-12-04T14:58:25,469 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:58:25,473 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T14:58:25,489 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-04T14:58:25,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:58:25,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36649-0x101a2238e4c0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:58:25,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741827_1003 (size=196) 2024-12-04T14:58:25,975 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:58:25,977 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T14:58:25,991 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:25,994 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T14:58:26,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741828_1004 (size=1189) 2024-12-04T14:58:26,445 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/data/master/store 2024-12-04T14:58:26,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741829_1005 (size=34) 2024-12-04T14:58:26,466 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-04T14:58:26,469 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:58:26,470 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T14:58:26,470 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:58:26,470 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:58:26,472 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T14:58:26,472 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:58:26,472 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:58:26,473 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733324306470Disabling compacts and flushes for region at 1733324306470Disabling writes for close at 1733324306472 (+2 ms)Writing region close event to WAL at 1733324306472Closed at 1733324306472 2024-12-04T14:58:26,476 WARN [master/d214edae7bd2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/data/master/store/.initializing 2024-12-04T14:58:26,476 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/WALs/d214edae7bd2,39449,1733324304115 2024-12-04T14:58:26,483 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T14:58:26,497 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d214edae7bd2%2C39449%2C1733324304115, suffix=, logDir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/WALs/d214edae7bd2,39449,1733324304115, archiveDir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/oldWALs, maxLogs=10 2024-12-04T14:58:26,516 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/WALs/d214edae7bd2,39449,1733324304115/d214edae7bd2%2C39449%2C1733324304115.1733324306501, exclude list is [], retry=0 2024-12-04T14:58:26,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44941,DS-40b04e37-5102-422a-9b3a-9b86db1d47b1,DISK] 2024-12-04T14:58:26,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-04T14:58:26,560 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/WALs/d214edae7bd2,39449,1733324304115/d214edae7bd2%2C39449%2C1733324304115.1733324306501 2024-12-04T14:58:26,561 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41829:41829)] 2024-12-04T14:58:26,561 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:58:26,562 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:58:26,564 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:58:26,565 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:58:26,597 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:58:26,619 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T14:58:26,622 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:26,624 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:58:26,625 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:58:26,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T14:58:26,628 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:26,629 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:58:26,630 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:58:26,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T14:58:26,632 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:26,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:58:26,634 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:58:26,636 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T14:58:26,637 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:26,638 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:58:26,638 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:58:26,641 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:58:26,642 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:58:26,647 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:58:26,648 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:58:26,651 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T14:58:26,655 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:58:26,660 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:58:26,661 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59423832, jitterRate=-0.11451590061187744}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T14:58:26,667 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733324306575Initializing all the Stores at 1733324306577 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733324306578 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324306578Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324306579 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324306579Cleaning up temporary data from old regions at 1733324306648 (+69 ms)Region opened successfully at 1733324306667 (+19 ms) 2024-12-04T14:58:26,668 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T14:58:26,698 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5878f5d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d214edae7bd2/172.17.0.2:0 2024-12-04T14:58:26,726 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T14:58:26,735 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T14:58:26,736 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T14:58:26,738 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T14:58:26,739 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-04T14:58:26,744 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-04T14:58:26,745 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T14:58:26,770 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T14:58:26,778 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T14:58:26,792 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T14:58:26,794 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T14:58:26,796 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T14:58:26,800 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T14:58:26,802 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T14:58:26,807 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T14:58:26,817 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T14:58:26,820 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T14:58:26,825 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T14:58:26,845 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T14:58:26,850 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T14:58:26,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T14:58:26,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36649-0x101a2238e4c0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T14:58:26,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:58:26,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36649-0x101a2238e4c0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:58:26,862 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d214edae7bd2,39449,1733324304115, sessionid=0x101a2238e4c0000, setting cluster-up flag (Was=false) 2024-12-04T14:58:26,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36649-0x101a2238e4c0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:58:26,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:58:26,909 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T14:58:26,912 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d214edae7bd2,39449,1733324304115 2024-12-04T14:58:26,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36649-0x101a2238e4c0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:58:26,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:58:26,959 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T14:58:26,961 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d214edae7bd2,39449,1733324304115 2024-12-04T14:58:26,970 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T14:58:26,986 INFO [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(746): ClusterId : e66e92f8-1d08-41ec-82b1-ea6168d3b6c4 2024-12-04T14:58:26,988 DEBUG [RS:0;d214edae7bd2:36649 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T14:58:27,002 DEBUG [RS:0;d214edae7bd2:36649 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T14:58:27,002 DEBUG [RS:0;d214edae7bd2:36649 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T14:58:27,010 DEBUG [RS:0;d214edae7bd2:36649 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T14:58:27,011 DEBUG [RS:0;d214edae7bd2:36649 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73c10c7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d214edae7bd2/172.17.0.2:0 2024-12-04T14:58:27,023 DEBUG [RS:0;d214edae7bd2:36649 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d214edae7bd2:36649 2024-12-04T14:58:27,026 INFO [RS:0;d214edae7bd2:36649 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T14:58:27,026 INFO [RS:0;d214edae7bd2:36649 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T14:58:27,026 DEBUG [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T14:58:27,029 INFO [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(2659): reportForDuty to master=d214edae7bd2,39449,1733324304115 with port=36649, startcode=1733324304801 2024-12-04T14:58:27,036 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T14:58:27,038 DEBUG [RS:0;d214edae7bd2:36649 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T14:58:27,044 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T14:58:27,051 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T14:58:27,057 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d214edae7bd2,39449,1733324304115 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T14:58:27,064 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d214edae7bd2:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:58:27,064 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d214edae7bd2:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:58:27,065 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d214edae7bd2:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:58:27,065 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d214edae7bd2:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:58:27,065 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d214edae7bd2:0, corePoolSize=10, maxPoolSize=10 2024-12-04T14:58:27,065 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:58:27,065 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d214edae7bd2:0, corePoolSize=2, maxPoolSize=2 2024-12-04T14:58:27,066 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:58:27,071 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733324337071 2024-12-04T14:58:27,072 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36577, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T14:58:27,072 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T14:58:27,072 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T14:58:27,073 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T14:58:27,074 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T14:58:27,077 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T14:58:27,077 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T14:58:27,077 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T14:58:27,078 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T14:58:27,077 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39449 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:27,078 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:27,078 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T14:58:27,078 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T14:58:27,081 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T14:58:27,082 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T14:58:27,083 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T14:58:27,085 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T14:58:27,085 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T14:58:27,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741831_1007 (size=1321) 2024-12-04T14:58:27,090 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d214edae7bd2:0:becomeActiveMaster-HFileCleaner.large.0-1733324307087,5,FailOnTimeoutGroup] 2024-12-04T14:58:27,093 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d214edae7bd2:0:becomeActiveMaster-HFileCleaner.small.0-1733324307091,5,FailOnTimeoutGroup] 2024-12-04T14:58:27,093 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T14:58:27,094 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T14:58:27,095 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T14:58:27,095 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T14:58:27,102 DEBUG [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-04T14:58:27,102 WARN [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-04T14:58:27,204 INFO [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(2659): reportForDuty to master=d214edae7bd2,39449,1733324304115 with port=36649, startcode=1733324304801 2024-12-04T14:58:27,209 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39449 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d214edae7bd2,36649,1733324304801 2024-12-04T14:58:27,215 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39449 {}] master.ServerManager(517): Registering regionserver=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:27,223 DEBUG [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee 2024-12-04T14:58:27,223 DEBUG [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39975 2024-12-04T14:58:27,223 DEBUG [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T14:58:27,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T14:58:27,234 DEBUG [RS:0;d214edae7bd2:36649 {}] zookeeper.ZKUtil(111): regionserver:36649-0x101a2238e4c0001, quorum=127.0.0.1:52559, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d214edae7bd2,36649,1733324304801 2024-12-04T14:58:27,234 WARN [RS:0;d214edae7bd2:36649 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T14:58:27,235 INFO [RS:0;d214edae7bd2:36649 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T14:58:27,235 DEBUG [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/WALs/d214edae7bd2,36649,1733324304801 2024-12-04T14:58:27,237 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d214edae7bd2,36649,1733324304801] 2024-12-04T14:58:27,260 INFO [RS:0;d214edae7bd2:36649 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T14:58:27,271 INFO [RS:0;d214edae7bd2:36649 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T14:58:27,276 INFO [RS:0;d214edae7bd2:36649 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T14:58:27,276 INFO [RS:0;d214edae7bd2:36649 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:58:27,277 INFO [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T14:58:27,281 INFO [RS:0;d214edae7bd2:36649 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T14:58:27,282 INFO [RS:0;d214edae7bd2:36649 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T14:58:27,283 DEBUG [RS:0;d214edae7bd2:36649 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:58:27,283 DEBUG [RS:0;d214edae7bd2:36649 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:58:27,283 DEBUG [RS:0;d214edae7bd2:36649 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:58:27,283 DEBUG [RS:0;d214edae7bd2:36649 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:58:27,283 DEBUG [RS:0;d214edae7bd2:36649 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:58:27,284 DEBUG [RS:0;d214edae7bd2:36649 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d214edae7bd2:0, corePoolSize=2, maxPoolSize=2 2024-12-04T14:58:27,284 DEBUG [RS:0;d214edae7bd2:36649 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:58:27,284 DEBUG [RS:0;d214edae7bd2:36649 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:58:27,284 DEBUG [RS:0;d214edae7bd2:36649 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:58:27,284 DEBUG [RS:0;d214edae7bd2:36649 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:58:27,284 DEBUG [RS:0;d214edae7bd2:36649 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:58:27,284 DEBUG [RS:0;d214edae7bd2:36649 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:58:27,285 DEBUG [RS:0;d214edae7bd2:36649 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d214edae7bd2:0, corePoolSize=3, maxPoolSize=3 2024-12-04T14:58:27,285 DEBUG [RS:0;d214edae7bd2:36649 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0, corePoolSize=3, maxPoolSize=3 2024-12-04T14:58:27,285 INFO [RS:0;d214edae7bd2:36649 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T14:58:27,286 INFO [RS:0;d214edae7bd2:36649 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T14:58:27,286 INFO [RS:0;d214edae7bd2:36649 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:58:27,286 INFO [RS:0;d214edae7bd2:36649 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T14:58:27,286 INFO [RS:0;d214edae7bd2:36649 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T14:58:27,286 INFO [RS:0;d214edae7bd2:36649 {}] hbase.ChoreService(168): Chore ScheduledChore name=d214edae7bd2,36649,1733324304801-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T14:58:27,304 INFO [RS:0;d214edae7bd2:36649 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T14:58:27,306 INFO [RS:0;d214edae7bd2:36649 {}] hbase.ChoreService(168): Chore ScheduledChore name=d214edae7bd2,36649,1733324304801-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:58:27,306 INFO [RS:0;d214edae7bd2:36649 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:58:27,306 INFO [RS:0;d214edae7bd2:36649 {}] regionserver.Replication(171): d214edae7bd2,36649,1733324304801 started 2024-12-04T14:58:27,322 INFO [RS:0;d214edae7bd2:36649 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:58:27,323 INFO [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(1482): Serving as d214edae7bd2,36649,1733324304801, RpcServer on d214edae7bd2/172.17.0.2:36649, sessionid=0x101a2238e4c0001 2024-12-04T14:58:27,324 DEBUG [RS:0;d214edae7bd2:36649 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T14:58:27,324 DEBUG [RS:0;d214edae7bd2:36649 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d214edae7bd2,36649,1733324304801 2024-12-04T14:58:27,324 DEBUG [RS:0;d214edae7bd2:36649 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd214edae7bd2,36649,1733324304801' 2024-12-04T14:58:27,324 DEBUG [RS:0;d214edae7bd2:36649 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T14:58:27,325 DEBUG [RS:0;d214edae7bd2:36649 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T14:58:27,326 DEBUG [RS:0;d214edae7bd2:36649 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T14:58:27,326 DEBUG [RS:0;d214edae7bd2:36649 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T14:58:27,326 DEBUG [RS:0;d214edae7bd2:36649 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d214edae7bd2,36649,1733324304801 2024-12-04T14:58:27,326 DEBUG [RS:0;d214edae7bd2:36649 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd214edae7bd2,36649,1733324304801' 2024-12-04T14:58:27,327 DEBUG [RS:0;d214edae7bd2:36649 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T14:58:27,327 DEBUG [RS:0;d214edae7bd2:36649 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T14:58:27,328 DEBUG [RS:0;d214edae7bd2:36649 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T14:58:27,328 INFO [RS:0;d214edae7bd2:36649 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T14:58:27,328 INFO [RS:0;d214edae7bd2:36649 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T14:58:27,438 INFO [RS:0;d214edae7bd2:36649 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T14:58:27,442 INFO [RS:0;d214edae7bd2:36649 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d214edae7bd2%2C36649%2C1733324304801, suffix=, logDir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/WALs/d214edae7bd2,36649,1733324304801, archiveDir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/oldWALs, maxLogs=32 2024-12-04T14:58:27,458 DEBUG [RS:0;d214edae7bd2:36649 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/WALs/d214edae7bd2,36649,1733324304801/d214edae7bd2%2C36649%2C1733324304801.1733324307444, exclude list is [], retry=0 2024-12-04T14:58:27,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44941,DS-40b04e37-5102-422a-9b3a-9b86db1d47b1,DISK] 2024-12-04T14:58:27,465 INFO [RS:0;d214edae7bd2:36649 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/WALs/d214edae7bd2,36649,1733324304801/d214edae7bd2%2C36649%2C1733324304801.1733324307444 2024-12-04T14:58:27,466 DEBUG [RS:0;d214edae7bd2:36649 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41829:41829)] 2024-12-04T14:58:27,490 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T14:58:27,491 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee 2024-12-04T14:58:27,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741833_1009 (size=32) 2024-12-04T14:58:27,906 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:58:27,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T14:58:27,911 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T14:58:27,911 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:27,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:58:27,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T14:58:27,914 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T14:58:27,914 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:27,915 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:58:27,915 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T14:58:27,917 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T14:58:27,917 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:27,918 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:58:27,918 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T14:58:27,920 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T14:58:27,920 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:27,921 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:58:27,921 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T14:58:27,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740 2024-12-04T14:58:27,924 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740 2024-12-04T14:58:27,926 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T14:58:27,926 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T14:58:27,927 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:58:27,930 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T14:58:27,934 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:58:27,935 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75077544, jitterRate=0.11874258518218994}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:58:27,937 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733324307906Initializing all the Stores at 1733324307908 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733324307908Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733324307908Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324307908Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733324307909 (+1 ms)Cleaning up temporary data from old regions at 1733324307927 (+18 ms)Region opened successfully at 1733324307937 (+10 ms) 2024-12-04T14:58:27,937 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T14:58:27,937 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T14:58:27,937 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T14:58:27,937 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T14:58:27,937 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T14:58:27,938 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T14:58:27,939 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733324307937Disabling compacts and flushes for region at 1733324307937Disabling writes for close at 1733324307937Writing region close event to WAL at 1733324307938 (+1 ms)Closed at 1733324307938 2024-12-04T14:58:27,941 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T14:58:27,941 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T14:58:27,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T14:58:27,954 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T14:58:27,956 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T14:58:28,115 DEBUG [d214edae7bd2:39449 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T14:58:28,127 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:28,133 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d214edae7bd2,36649,1733324304801, state=OPENING 2024-12-04T14:58:28,175 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T14:58:28,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:58:28,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36649-0x101a2238e4c0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:58:28,185 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:58:28,185 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:58:28,188 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T14:58:28,191 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d214edae7bd2,36649,1733324304801}] 2024-12-04T14:58:28,372 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T14:58:28,375 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43503, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T14:58:28,385 INFO [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T14:58:28,385 INFO [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T14:58:28,385 INFO [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-04T14:58:28,388 INFO [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d214edae7bd2%2C36649%2C1733324304801.meta, suffix=.meta, logDir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/WALs/d214edae7bd2,36649,1733324304801, archiveDir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/oldWALs, maxLogs=32 2024-12-04T14:58:28,401 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/WALs/d214edae7bd2,36649,1733324304801/d214edae7bd2%2C36649%2C1733324304801.meta.1733324308390.meta, exclude list is [], retry=0 2024-12-04T14:58:28,405 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44941,DS-40b04e37-5102-422a-9b3a-9b86db1d47b1,DISK] 2024-12-04T14:58:28,407 INFO [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/WALs/d214edae7bd2,36649,1733324304801/d214edae7bd2%2C36649%2C1733324304801.meta.1733324308390.meta 2024-12-04T14:58:28,408 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41829:41829)] 2024-12-04T14:58:28,408 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:58:28,410 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T14:58:28,412 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T14:58:28,416 INFO [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T14:58:28,420 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T14:58:28,420 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:58:28,420 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T14:58:28,420 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T14:58:28,424 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T14:58:28,426 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T14:58:28,426 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:28,427 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:58:28,427 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T14:58:28,429 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T14:58:28,429 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:28,430 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:58:28,430 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T14:58:28,432 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T14:58:28,432 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:28,433 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:58:28,433 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T14:58:28,434 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T14:58:28,434 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:28,435 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:58:28,435 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T14:58:28,437 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740 2024-12-04T14:58:28,440 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740 2024-12-04T14:58:28,443 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T14:58:28,444 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T14:58:28,445 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:58:28,448 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T14:58:28,450 INFO [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59928463, jitterRate=-0.1069963127374649}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:58:28,450 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T14:58:28,451 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733324308421Writing region info on filesystem at 1733324308421Initializing all the Stores at 1733324308423 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733324308423Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733324308424 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324308424Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733324308424Cleaning up temporary data from old regions at 1733324308444 (+20 ms)Running coprocessor post-open hooks at 1733324308450 (+6 ms)Region opened successfully at 1733324308451 (+1 ms) 2024-12-04T14:58:28,457 INFO [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733324308364 2024-12-04T14:58:28,467 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T14:58:28,467 INFO [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T14:58:28,468 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:28,471 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d214edae7bd2,36649,1733324304801, state=OPEN 2024-12-04T14:58:28,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36649-0x101a2238e4c0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T14:58:28,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T14:58:28,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:58:28,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:58:28,501 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:28,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T14:58:28,510 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d214edae7bd2,36649,1733324304801 in 310 msec 2024-12-04T14:58:28,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T14:58:28,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 566 msec 2024-12-04T14:58:28,518 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T14:58:28,518 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T14:58:28,535 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:28,536 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:58:28,553 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:28,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53325, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:28,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5760 sec 2024-12-04T14:58:28,574 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733324308574, completionTime=-1 2024-12-04T14:58:28,578 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T14:58:28,578 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T14:58:28,607 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T14:58:28,607 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733324368607 2024-12-04T14:58:28,608 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733324428607 2024-12-04T14:58:28,608 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-12-04T14:58:28,610 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d214edae7bd2,39449,1733324304115-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:58:28,611 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d214edae7bd2,39449,1733324304115-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:58:28,611 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d214edae7bd2,39449,1733324304115-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:58:28,612 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d214edae7bd2:39449, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:58:28,613 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T14:58:28,613 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T14:58:28,618 DEBUG [master/d214edae7bd2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T14:58:28,640 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.689sec 2024-12-04T14:58:28,641 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T14:58:28,642 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T14:58:28,642 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T14:58:28,643 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T14:58:28,643 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T14:58:28,644 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d214edae7bd2,39449,1733324304115-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T14:58:28,645 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d214edae7bd2,39449,1733324304115-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T14:58:28,652 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T14:58:28,653 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T14:58:28,654 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d214edae7bd2,39449,1733324304115-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:58:28,699 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c90ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:28,702 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-04T14:58:28,702 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-04T14:58:28,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:58:28,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:28,713 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:58:28,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:28,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:58:28,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24f079a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:28,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:58:28,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:28,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:28,724 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:28,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ba7d24a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:28,727 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:58:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:28,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:28,755 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d214edae7bd2,39449,1733324304115 2024-12-04T14:58:28,766 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=473, ProcessCount=11, AvailableMemoryMB=9480 2024-12-04T14:58:28,789 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T14:58:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is d214edae7bd2,39449,1733324304115 2024-12-04T14:58:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@453088aa 2024-12-04T14:58:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T14:58:28,803 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43886, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T14:58:28,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T14:58:28,809 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:58:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T14:58:28,824 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T14:58:28,826 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-04T14:58:28,826 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:28,828 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T14:58:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T14:58:28,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741835_1011 (size=1061) 2024-12-04T14:58:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T14:58:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T14:58:29,250 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee 2024-12-04T14:58:29,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741836_1012 (size=53) 2024-12-04T14:58:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T14:58:29,662 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:58:29,662 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7fc4055584695509eee4923dfb5273a9, disabling compactions & flushes 2024-12-04T14:58:29,662 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:29,662 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:29,663 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. after waiting 0 ms 2024-12-04T14:58:29,663 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:29,663 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:29,663 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7fc4055584695509eee4923dfb5273a9: Waiting for close lock at 1733324309662Disabling compacts and flushes for region at 1733324309662Disabling writes for close at 1733324309663 (+1 ms)Writing region close event to WAL at 1733324309663Closed at 1733324309663 2024-12-04T14:58:29,666 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T14:58:29,672 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733324309666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733324309666"}]},"ts":"1733324309666"} 2024-12-04T14:58:29,677 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T14:58:29,679 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T14:58:29,681 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324309679"}]},"ts":"1733324309679"} 2024-12-04T14:58:29,686 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T14:58:29,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fc4055584695509eee4923dfb5273a9, ASSIGN}] 2024-12-04T14:58:29,690 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fc4055584695509eee4923dfb5273a9, ASSIGN 2024-12-04T14:58:29,692 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fc4055584695509eee4923dfb5273a9, ASSIGN; state=OFFLINE, location=d214edae7bd2,36649,1733324304801; forceNewPlan=false, retain=false 2024-12-04T14:58:29,844 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7fc4055584695509eee4923dfb5273a9, regionState=OPENING, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:29,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fc4055584695509eee4923dfb5273a9, ASSIGN because future has completed 2024-12-04T14:58:29,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801}] 2024-12-04T14:58:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T14:58:30,020 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:30,020 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:58:30,020 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:30,020 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:58:30,020 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:30,020 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:30,023 INFO [StoreOpener-7fc4055584695509eee4923dfb5273a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:30,026 INFO [StoreOpener-7fc4055584695509eee4923dfb5273a9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T14:58:30,026 INFO [StoreOpener-7fc4055584695509eee4923dfb5273a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fc4055584695509eee4923dfb5273a9 columnFamilyName A 2024-12-04T14:58:30,026 DEBUG [StoreOpener-7fc4055584695509eee4923dfb5273a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:30,027 INFO [StoreOpener-7fc4055584695509eee4923dfb5273a9-1 {}] regionserver.HStore(327): Store=7fc4055584695509eee4923dfb5273a9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:58:30,028 INFO [StoreOpener-7fc4055584695509eee4923dfb5273a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:30,030 INFO [StoreOpener-7fc4055584695509eee4923dfb5273a9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T14:58:30,030 INFO [StoreOpener-7fc4055584695509eee4923dfb5273a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fc4055584695509eee4923dfb5273a9 columnFamilyName B 2024-12-04T14:58:30,030 DEBUG [StoreOpener-7fc4055584695509eee4923dfb5273a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:30,031 INFO [StoreOpener-7fc4055584695509eee4923dfb5273a9-1 {}] regionserver.HStore(327): Store=7fc4055584695509eee4923dfb5273a9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:58:30,031 INFO [StoreOpener-7fc4055584695509eee4923dfb5273a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:30,033 INFO [StoreOpener-7fc4055584695509eee4923dfb5273a9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T14:58:30,033 INFO [StoreOpener-7fc4055584695509eee4923dfb5273a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fc4055584695509eee4923dfb5273a9 columnFamilyName C 2024-12-04T14:58:30,033 DEBUG [StoreOpener-7fc4055584695509eee4923dfb5273a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:30,034 INFO [StoreOpener-7fc4055584695509eee4923dfb5273a9-1 {}] regionserver.HStore(327): Store=7fc4055584695509eee4923dfb5273a9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:58:30,035 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:30,035 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:30,036 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:30,037 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:30,038 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:30,039 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:30,039 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:58:30,041 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:30,045 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:58:30,046 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7fc4055584695509eee4923dfb5273a9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68539238, jitterRate=0.021314233541488647}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:58:30,046 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:30,047 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7fc4055584695509eee4923dfb5273a9: Running coprocessor pre-open hook at 1733324310021Writing region info on filesystem at 1733324310021Initializing all the Stores at 1733324310022 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324310022Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324310022Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324310023 (+1 ms)Cleaning up temporary data from old regions at 1733324310039 (+16 ms)Running coprocessor post-open hooks at 1733324310046 (+7 ms)Region opened successfully at 1733324310047 (+1 ms) 2024-12-04T14:58:30,049 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., pid=6, masterSystemTime=1733324310009 2024-12-04T14:58:30,053 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:30,053 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:30,055 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7fc4055584695509eee4923dfb5273a9, regionState=OPEN, openSeqNum=2, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:30,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 because future has completed 2024-12-04T14:58:30,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T14:58:30,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 in 208 msec 2024-12-04T14:58:30,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T14:58:30,072 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fc4055584695509eee4923dfb5273a9, ASSIGN in 380 msec 2024-12-04T14:58:30,073 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T14:58:30,073 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324310073"}]},"ts":"1733324310073"} 2024-12-04T14:58:30,076 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T14:58:30,078 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T14:58:30,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2620 sec 2024-12-04T14:58:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T14:58:30,984 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:30,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c8b07f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:30,992 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:58:30,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:30,995 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:58:30,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:30,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:58:30,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72ddcdb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:30,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:58:30,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:30,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:30,999 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6924ee0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,001 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:31,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:58:31,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:31,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:31,007 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T14:58:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is d214edae7bd2,39449,1733324304115 2024-12-04T14:58:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6d6e054b 2024-12-04T14:58:31,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T14:58:31,014 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T14:58:31,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff18d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:58:31,018 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:31,020 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:58:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:58:31,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@90ab23d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:58:31,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:31,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:31,023 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:31,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@643162ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109bd06b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,024 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:58:31,025 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:31,026 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:58:31,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:58:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a8011b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:58:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:31,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:31,029 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:31,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1380f017, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af20e9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,031 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:58:31,031 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:31,032 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:58:31,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:31,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:58:31,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6b748f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:58:31,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:31,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:31,036 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46fadb48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,037 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7387a89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,037 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:58:31,038 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:31,039 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:58:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:58:31,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63ac9955, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:58:31,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:31,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:31,042 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:31,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72f860d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75a031a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,044 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:58:31,044 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:31,045 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:58:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:58:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a5b3e13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:58:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:31,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:31,047 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:31,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@589b33a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,050 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@276a94e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,050 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:58:31,050 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:31,051 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:58:31,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:31,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:58:31,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@657524e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:58:31,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:31,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:31,053 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:31,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d5c59a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d403639, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,055 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:58:31,055 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:31,056 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:58:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:58:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c0a534c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:58:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:31,058 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58dec3db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52d66d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,060 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:58:31,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:31,062 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:58:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:58:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388f703a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:58:31,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:31,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:31,064 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c1b1a46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,066 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7be0d766, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,066 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:58:31,066 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:31,067 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:58:31,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:31,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:58:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c19ce67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:58:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:31,069 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31cfd059, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,073 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:31,073 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:31,075 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:31,075 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:31,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:58:31,075 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:31,076 DEBUG [Thread-158 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:58:31,076 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:58:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:58:31,077 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:31,078 DEBUG [Thread-158 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:31,079 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:58:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:31,080 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:31,080 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:31,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:31,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:31,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:31,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:58:31,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:58:31,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:31,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:31,084 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:31,084 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-04T14:58:31,086 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:31,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T14:58:31,087 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T14:58:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T14:58:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T14:58:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T14:58:31,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:58:31,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T14:58:31,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:31,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T14:58:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T14:58:31,092 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:31,094 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:58:31,094 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:31,096 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:31,097 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:31,097 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:31,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:31,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T14:58:31,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T14:58:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:31,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T14:58:31,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:31,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:31,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:31,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:31,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:31,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T14:58:31,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/14a96173a36348dd89478c112150b3a4 is 50, key is test_row_0/A:col10/1733324311133/Put/seqid=0 2024-12-04T14:58:31,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T14:58:31,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324371251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324371247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324371260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:31,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:31,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324371271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:31,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:31,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:31,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:31,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:31,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59310 deadline: 1733324371280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:31,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:31,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741837_1013 (size=12001) 2024-12-04T14:58:31,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:31,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/14a96173a36348dd89478c112150b3a4 2024-12-04T14:58:31,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:31,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324371392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:31,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324371393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:31,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324371393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:31,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324371393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59310 deadline: 1733324371401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T14:58:31,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/7a3881a81f83456ca46955f50f3a6312 is 50, key is test_row_0/B:col10/1733324311133/Put/seqid=0 2024-12-04T14:58:31,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741838_1014 (size=12001) 2024-12-04T14:58:31,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/7a3881a81f83456ca46955f50f3a6312 2024-12-04T14:58:31,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T14:58:31,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:31,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:31,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:31,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:31,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:31,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/607fd62f9ad04a08bcb0f4dfaf384af9 is 50, key is test_row_0/C:col10/1733324311133/Put/seqid=0 2024-12-04T14:58:31,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741839_1015 (size=12001) 2024-12-04T14:58:31,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/607fd62f9ad04a08bcb0f4dfaf384af9 2024-12-04T14:58:31,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/14a96173a36348dd89478c112150b3a4 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/14a96173a36348dd89478c112150b3a4 2024-12-04T14:58:31,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/14a96173a36348dd89478c112150b3a4, entries=150, sequenceid=13, filesize=11.7 K 2024-12-04T14:58:31,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/7a3881a81f83456ca46955f50f3a6312 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/7a3881a81f83456ca46955f50f3a6312 2024-12-04T14:58:31,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/7a3881a81f83456ca46955f50f3a6312, entries=150, sequenceid=13, filesize=11.7 K 2024-12-04T14:58:31,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/607fd62f9ad04a08bcb0f4dfaf384af9 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/607fd62f9ad04a08bcb0f4dfaf384af9 2024-12-04T14:58:31,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/607fd62f9ad04a08bcb0f4dfaf384af9, entries=150, sequenceid=13, filesize=11.7 K 2024-12-04T14:58:31,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 7fc4055584695509eee4923dfb5273a9 in 425ms, sequenceid=13, compaction requested=false 2024-12-04T14:58:31,584 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-04T14:58:31,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:31,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-12-04T14:58:31,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:31,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:31,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:31,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:31,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:31,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:31,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T14:58:31,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:31,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:31,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:31,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:31,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:31,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/35cce42dade84e019923fd284dc37edb is 50, key is test_row_0/A:col10/1733324311237/Put/seqid=0 2024-12-04T14:58:31,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324371651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:31,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324371655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:31,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59310 deadline: 1733324371656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:31,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741840_1016 (size=16681) 2024-12-04T14:58:31,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/35cce42dade84e019923fd284dc37edb 2024-12-04T14:58:31,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324371659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:31,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324371661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:31,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/3f843abf7a4246089691886922a9b5e4 is 50, key is test_row_0/B:col10/1733324311237/Put/seqid=0 2024-12-04T14:58:31,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T14:58:31,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741841_1017 (size=12001) 2024-12-04T14:58:31,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324371771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:31,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324371771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:31,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T14:58:31,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:31,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:31,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:31,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:31,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:31,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:31,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324371790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59310 deadline: 1733324371781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:31,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:31,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324371792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:31,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T14:58:31,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:31,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:31,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:31,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:31,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:31,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:31,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324371982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:31,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:31,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324371982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:31,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:32,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59310 deadline: 1733324372002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:32,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:32,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324372003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:32,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:32,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324372002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:32,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:32,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T14:58:32,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:32,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:32,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:32,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:32,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:32,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:32,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/3f843abf7a4246089691886922a9b5e4 2024-12-04T14:58:32,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/5a18909aa0794c4180e06dede9ada2ad is 50, key is test_row_0/C:col10/1733324311237/Put/seqid=0 2024-12-04T14:58:32,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741842_1018 (size=12001) 2024-12-04T14:58:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T14:58:32,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T14:58:32,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:32,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:32,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:32,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:32,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:32,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:32,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324372301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:32,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324372302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:32,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:32,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59310 deadline: 1733324372311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:32,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324372320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:32,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324372320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:32,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T14:58:32,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:32,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:32,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:32,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:32,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:32,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T14:58:32,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:32,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:32,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:32,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:32,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:32,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:32,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/5a18909aa0794c4180e06dede9ada2ad 2024-12-04T14:58:32,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/35cce42dade84e019923fd284dc37edb as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/35cce42dade84e019923fd284dc37edb 2024-12-04T14:58:32,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/35cce42dade84e019923fd284dc37edb, entries=250, sequenceid=35, filesize=16.3 K 2024-12-04T14:58:32,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/3f843abf7a4246089691886922a9b5e4 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/3f843abf7a4246089691886922a9b5e4 2024-12-04T14:58:32,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/3f843abf7a4246089691886922a9b5e4, entries=150, sequenceid=35, filesize=11.7 K 2024-12-04T14:58:32,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/5a18909aa0794c4180e06dede9ada2ad as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/5a18909aa0794c4180e06dede9ada2ad 2024-12-04T14:58:32,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/5a18909aa0794c4180e06dede9ada2ad, entries=150, sequenceid=35, filesize=11.7 K 2024-12-04T14:58:32,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 7fc4055584695509eee4923dfb5273a9 in 1073ms, sequenceid=35, compaction requested=false 2024-12-04T14:58:32,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:32,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T14:58:32,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:32,744 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T14:58:32,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:32,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:32,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:32,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:32,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:32,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:32,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/e76854045f1245a0bc2531322e4edee0 is 50, key is test_row_0/A:col10/1733324311643/Put/seqid=0 2024-12-04T14:58:32,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741843_1019 (size=12001) 2024-12-04T14:58:32,804 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/e76854045f1245a0bc2531322e4edee0 2024-12-04T14:58:32,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:32,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:32,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/48ba29afd30b49ba9a5605ef9ebbc0cf is 50, key is test_row_0/B:col10/1733324311643/Put/seqid=0 2024-12-04T14:58:32,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741844_1020 (size=12001) 2024-12-04T14:58:32,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=109, cellSize=4985, totalHeapSize=11576, min timestamp=1733324312888, max timestamp=1733324312893Num uniques 859; 2024-12-04T14:58:32,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=4985, suffix heap size=117064,new segment heap size=11576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=109 2024-12-04T14:58:32,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=100, cellSize=4580, totalHeapSize=10640, min timestamp=1733324312888, max timestamp=1733324312893Num uniques 859; 2024-12-04T14:58:32,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=4580, suffix heap size=117064,new segment heap size=10640 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=100 2024-12-04T14:58:32,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=109, cellSize=4985, totalHeapSize=11576, min timestamp=1733324312888, max timestamp=1733324312905Num uniques 859; 2024-12-04T14:58:32,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=4985, suffix heap size=117064,new segment heap size=11576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=109 2024-12-04T14:58:33,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733324313056, max timestamp=1733324313089Num uniques 968; 2024-12-04T14:58:33,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=44328, new segment data size=7688, suffix heap size=128640,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=968, new segment cells count=168 2024-12-04T14:58:33,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733324313056, max timestamp=1733324313091Num uniques 959; 2024-12-04T14:58:33,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=43923, new segment data size=6870, suffix heap size=127704,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=959, new segment cells count=150 2024-12-04T14:58:33,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733324313056, max timestamp=1733324313099Num uniques 968; 2024-12-04T14:58:33,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=44328, new segment data size=7688, suffix heap size=128640,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=968, new segment cells count=168 2024-12-04T14:58:33,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59310 deadline: 1733324373187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:33,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:33,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324373188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:33,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:33,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324373190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:33,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324373198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:33,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324373215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:33,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T14:58:33,256 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T14:58:33,276 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/48ba29afd30b49ba9a5605ef9ebbc0cf 2024-12-04T14:58:33,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/de90ccc7605e41eeae431b85bd479d1e is 50, key is test_row_0/C:col10/1733324311643/Put/seqid=0 2024-12-04T14:58:33,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59310 deadline: 1733324373321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:33,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:33,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324373321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:33,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324373331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:33,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:33,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324373321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:33,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:33,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324373331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:33,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741845_1021 (size=12001) 2024-12-04T14:58:33,378 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/de90ccc7605e41eeae431b85bd479d1e 2024-12-04T14:58:33,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/e76854045f1245a0bc2531322e4edee0 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e76854045f1245a0bc2531322e4edee0 2024-12-04T14:58:33,409 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e76854045f1245a0bc2531322e4edee0, entries=150, sequenceid=45, filesize=11.7 K 2024-12-04T14:58:33,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/48ba29afd30b49ba9a5605ef9ebbc0cf as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/48ba29afd30b49ba9a5605ef9ebbc0cf 2024-12-04T14:58:33,431 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/48ba29afd30b49ba9a5605ef9ebbc0cf, entries=150, sequenceid=45, filesize=11.7 K 2024-12-04T14:58:33,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/de90ccc7605e41eeae431b85bd479d1e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/de90ccc7605e41eeae431b85bd479d1e 2024-12-04T14:58:33,454 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/de90ccc7605e41eeae431b85bd479d1e, entries=150, sequenceid=45, filesize=11.7 K 2024-12-04T14:58:33,467 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.38 KB/136582 for 7fc4055584695509eee4923dfb5273a9 in 722ms, sequenceid=45, compaction requested=true 2024-12-04T14:58:33,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:33,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:33,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-04T14:58:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-04T14:58:33,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-04T14:58:33,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3730 sec 2024-12-04T14:58:33,480 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.3990 sec 2024-12-04T14:58:33,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733324313156, max timestamp=1733324313562Num uniques 1009; 2024-12-04T14:58:33,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-12-04T14:58:33,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733324313156, max timestamp=1733324313565Num uniques 1027; 2024-12-04T14:58:33,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-04T14:58:33,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:33,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=78.27 KB heapSize=217.80 KB 2024-12-04T14:58:33,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733324313156, max timestamp=1733324313559Num uniques 1027; 2024-12-04T14:58:33,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-04T14:58:33,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:33,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:33,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:33,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:33,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:33,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:33,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/b35c0ad0440e47dfa60b517fce5988d8 is 50, key is test_row_0/A:col10/1733324313571/Put/seqid=0 2024-12-04T14:58:33,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741846_1022 (size=14341) 2024-12-04T14:58:33,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/b35c0ad0440e47dfa60b517fce5988d8 2024-12-04T14:58:33,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/a5c295b156764dc2b75a34692cfed6f1 is 50, key is test_row_0/B:col10/1733324313571/Put/seqid=0 2024-12-04T14:58:33,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733324313619, max timestamp=1733324313660Num uniques 859; 2024-12-04T14:58:33,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T14:58:33,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733324313619, max timestamp=1733324313659Num uniques 859; 2024-12-04T14:58:33,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733324313619, max timestamp=1733324313659Num uniques 859; 2024-12-04T14:58:33,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T14:58:33,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T14:58:33,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741847_1023 (size=12001) 2024-12-04T14:58:33,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/a5c295b156764dc2b75a34692cfed6f1 2024-12-04T14:58:33,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324373716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:33,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324373721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59310 deadline: 1733324373717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:33,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324373724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:33,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/74c6b81fc322453890bab73ca1c86048 is 50, key is test_row_0/C:col10/1733324313571/Put/seqid=0 2024-12-04T14:58:33,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:33,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324373736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:33,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:33,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741848_1024 (size=12001) 2024-12-04T14:58:33,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/74c6b81fc322453890bab73ca1c86048 2024-12-04T14:58:33,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/b35c0ad0440e47dfa60b517fce5988d8 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/b35c0ad0440e47dfa60b517fce5988d8 2024-12-04T14:58:33,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/b35c0ad0440e47dfa60b517fce5988d8, entries=200, sequenceid=103, filesize=14.0 K 2024-12-04T14:58:33,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/a5c295b156764dc2b75a34692cfed6f1 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/a5c295b156764dc2b75a34692cfed6f1 2024-12-04T14:58:33,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/a5c295b156764dc2b75a34692cfed6f1, entries=150, sequenceid=103, filesize=11.7 K 2024-12-04T14:58:33,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59310 deadline: 1733324373845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:33,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:33,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/74c6b81fc322453890bab73ca1c86048 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/74c6b81fc322453890bab73ca1c86048 2024-12-04T14:58:33,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324373845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324373844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:33,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324373857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:33,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:33,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324373855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:33,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/74c6b81fc322453890bab73ca1c86048, entries=150, sequenceid=103, filesize=11.7 K 2024-12-04T14:58:33,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~124.27 KB/127248, currentSize=140.89 KB/144270 for 7fc4055584695509eee4923dfb5273a9 in 307ms, sequenceid=103, compaction requested=true 2024-12-04T14:58:33,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:33,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:33,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:33,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:33,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:33,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:33,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:33,895 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:58:33,902 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:58:33,903 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:58:33,904 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/B is initiating minor compaction (all files) 2024-12-04T14:58:33,906 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/B in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:33,907 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/7a3881a81f83456ca46955f50f3a6312, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/3f843abf7a4246089691886922a9b5e4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/48ba29afd30b49ba9a5605ef9ebbc0cf, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/a5c295b156764dc2b75a34692cfed6f1] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=46.9 K 2024-12-04T14:58:33,908 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55024 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:58:33,908 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/A is initiating minor compaction (all files) 2024-12-04T14:58:33,909 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/A in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:33,909 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/14a96173a36348dd89478c112150b3a4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/35cce42dade84e019923fd284dc37edb, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e76854045f1245a0bc2531322e4edee0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/b35c0ad0440e47dfa60b517fce5988d8] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=53.7 K 2024-12-04T14:58:33,909 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a3881a81f83456ca46955f50f3a6312, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733324311133 2024-12-04T14:58:33,911 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f843abf7a4246089691886922a9b5e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1733324311237 2024-12-04T14:58:33,911 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14a96173a36348dd89478c112150b3a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733324311133 2024-12-04T14:58:33,914 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35cce42dade84e019923fd284dc37edb, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1733324311237 2024-12-04T14:58:33,914 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 48ba29afd30b49ba9a5605ef9ebbc0cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733324311642 2024-12-04T14:58:33,920 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e76854045f1245a0bc2531322e4edee0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733324311642 2024-12-04T14:58:33,920 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting a5c295b156764dc2b75a34692cfed6f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733324313559 2024-12-04T14:58:33,921 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting b35c0ad0440e47dfa60b517fce5988d8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733324313559 2024-12-04T14:58:33,939 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T14:58:33,988 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#B#compaction#13 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:33,988 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#A#compaction#12 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:33,989 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/70e9cf57722045eca89793b53cab3175 is 50, key is test_row_0/A:col10/1733324313571/Put/seqid=0 2024-12-04T14:58:33,997 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/49e84e9317a9494083825557acbe6f0e is 50, key is test_row_0/B:col10/1733324313571/Put/seqid=0 2024-12-04T14:58:34,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741849_1025 (size=12139) 2024-12-04T14:58:34,062 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/70e9cf57722045eca89793b53cab3175 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/70e9cf57722045eca89793b53cab3175 2024-12-04T14:58:34,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741850_1026 (size=12139) 2024-12-04T14:58:34,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:34,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-04T14:58:34,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:34,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:34,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:34,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:34,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:34,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:34,087 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/A of 7fc4055584695509eee4923dfb5273a9 into 70e9cf57722045eca89793b53cab3175(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:34,087 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:34,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/e4cfb410f2a347428b72589277869c0f is 50, key is test_row_0/A:col10/1733324313732/Put/seqid=0 2024-12-04T14:58:34,102 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/A, priority=12, startTime=1733324313881; duration=0sec 2024-12-04T14:58:34,102 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:34,103 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:A 2024-12-04T14:58:34,103 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:58:34,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324374082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,110 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:58:34,110 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/C is initiating minor compaction (all files) 2024-12-04T14:58:34,110 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/C in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:34,110 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/607fd62f9ad04a08bcb0f4dfaf384af9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/5a18909aa0794c4180e06dede9ada2ad, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/de90ccc7605e41eeae431b85bd479d1e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/74c6b81fc322453890bab73ca1c86048] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=46.9 K 2024-12-04T14:58:34,111 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 607fd62f9ad04a08bcb0f4dfaf384af9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733324311133 2024-12-04T14:58:34,112 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a18909aa0794c4180e06dede9ada2ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1733324311237 2024-12-04T14:58:34,114 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting de90ccc7605e41eeae431b85bd479d1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733324311642 2024-12-04T14:58:34,115 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74c6b81fc322453890bab73ca1c86048, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733324313559 2024-12-04T14:58:34,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741851_1027 (size=14491) 2024-12-04T14:58:34,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/e4cfb410f2a347428b72589277869c0f 2024-12-04T14:58:34,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324374097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324374101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324374106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59310 deadline: 1733324374103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/db11f686763a4ffc93db4329e7975eeb is 50, key is test_row_0/B:col10/1733324313732/Put/seqid=0 2024-12-04T14:58:34,169 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#C#compaction#15 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:34,169 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/31afd7f434574f1aac5de6af5b7c1016 is 50, key is test_row_0/C:col10/1733324313571/Put/seqid=0 2024-12-04T14:58:34,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741852_1028 (size=12101) 2024-12-04T14:58:34,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/db11f686763a4ffc93db4329e7975eeb 2024-12-04T14:58:34,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324374220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741853_1029 (size=12139) 2024-12-04T14:58:34,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/36cea9cefb7044a1a68faa819a0d9149 is 50, key is test_row_0/C:col10/1733324313732/Put/seqid=0 2024-12-04T14:58:34,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324374251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324374251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,265 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/31afd7f434574f1aac5de6af5b7c1016 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/31afd7f434574f1aac5de6af5b7c1016 2024-12-04T14:58:34,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324374260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,279 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/C of 7fc4055584695509eee4923dfb5273a9 into 31afd7f434574f1aac5de6af5b7c1016(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:34,279 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:34,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,279 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/C, priority=12, startTime=1733324313893; duration=0sec 2024-12-04T14:58:34,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,279 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:34,279 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:C 2024-12-04T14:58:34,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59310 deadline: 1733324374260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741854_1030 (size=12101) 2024-12-04T14:58:34,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/36cea9cefb7044a1a68faa819a0d9149 2024-12-04T14:58:34,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/e4cfb410f2a347428b72589277869c0f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e4cfb410f2a347428b72589277869c0f 2024-12-04T14:58:34,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e4cfb410f2a347428b72589277869c0f, entries=200, sequenceid=140, filesize=14.2 K 2024-12-04T14:58:34,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/db11f686763a4ffc93db4329e7975eeb as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/db11f686763a4ffc93db4329e7975eeb 2024-12-04T14:58:34,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/db11f686763a4ffc93db4329e7975eeb, entries=150, sequenceid=140, filesize=11.8 K 2024-12-04T14:58:34,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/36cea9cefb7044a1a68faa819a0d9149 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/36cea9cefb7044a1a68faa819a0d9149 2024-12-04T14:58:34,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/36cea9cefb7044a1a68faa819a0d9149, entries=150, sequenceid=140, filesize=11.8 K 2024-12-04T14:58:34,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 7fc4055584695509eee4923dfb5273a9 in 270ms, sequenceid=140, compaction requested=false 2024-12-04T14:58:34,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:34,439 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T14:58:34,439 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T14:58:34,442 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T14:58:34,442 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T14:58:34,442 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-04T14:58:34,442 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-04T14:58:34,444 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-04T14:58:34,444 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-04T14:58:34,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:34,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T14:58:34,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:34,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:34,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:34,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:34,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:34,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:34,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/2a48cb6ad136411d879bde76f0f18354 is 50, key is test_row_0/A:col10/1733324314102/Put/seqid=0 2024-12-04T14:58:34,494 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/49e84e9317a9494083825557acbe6f0e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/49e84e9317a9494083825557acbe6f0e 2024-12-04T14:58:34,508 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/B of 7fc4055584695509eee4923dfb5273a9 into 49e84e9317a9494083825557acbe6f0e(size=11.9 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:34,509 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:34,509 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/B, priority=12, startTime=1733324313892; duration=0sec 2024-12-04T14:58:34,509 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:34,509 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:B 2024-12-04T14:58:34,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741855_1031 (size=16931) 2024-12-04T14:58:34,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/2a48cb6ad136411d879bde76f0f18354 2024-12-04T14:58:34,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/af1c86342cb14b9d89e4d4c73972a97b is 50, key is test_row_0/B:col10/1733324314102/Put/seqid=0 2024-12-04T14:58:34,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324314539, max timestamp=1733324314560Num uniques 859; 2024-12-04T14:58:34,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324314539, max timestamp=1733324314560Num uniques 859; 2024-12-04T14:58:34,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324314539, max timestamp=1733324314560Num uniques 859; 2024-12-04T14:58:34,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:58:34,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:58:34,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:58:34,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741856_1032 (size=12151) 2024-12-04T14:58:34,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324374611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59310 deadline: 1733324374621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324374621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324374624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324374627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324374730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324374751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324374752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324374752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59310 deadline: 1733324374751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324374940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324374961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324374961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324374963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:34,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/af1c86342cb14b9d89e4d4c73972a97b 2024-12-04T14:58:34,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59310 deadline: 1733324374981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:35,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/f3fe9f09221e41a58027a23ca71e9211 is 50, key is test_row_0/C:col10/1733324314102/Put/seqid=0 2024-12-04T14:58:35,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741857_1033 (size=12151) 2024-12-04T14:58:35,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/f3fe9f09221e41a58027a23ca71e9211 2024-12-04T14:58:35,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/2a48cb6ad136411d879bde76f0f18354 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/2a48cb6ad136411d879bde76f0f18354 2024-12-04T14:58:35,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/2a48cb6ad136411d879bde76f0f18354, entries=250, sequenceid=152, filesize=16.5 K 2024-12-04T14:58:35,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/af1c86342cb14b9d89e4d4c73972a97b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/af1c86342cb14b9d89e4d4c73972a97b 2024-12-04T14:58:35,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/af1c86342cb14b9d89e4d4c73972a97b, entries=150, sequenceid=152, filesize=11.9 K 2024-12-04T14:58:35,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/f3fe9f09221e41a58027a23ca71e9211 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/f3fe9f09221e41a58027a23ca71e9211 2024-12-04T14:58:35,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/f3fe9f09221e41a58027a23ca71e9211, entries=150, sequenceid=152, filesize=11.9 K 2024-12-04T14:58:35,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7fc4055584695509eee4923dfb5273a9 in 688ms, sequenceid=152, compaction requested=true 2024-12-04T14:58:35,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:35,147 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:35,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:35,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:35,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:35,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:35,148 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:35,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:35,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:35,150 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36391 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:35,151 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/B is initiating minor compaction (all files) 2024-12-04T14:58:35,151 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/B in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:35,151 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/49e84e9317a9494083825557acbe6f0e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/db11f686763a4ffc93db4329e7975eeb, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/af1c86342cb14b9d89e4d4c73972a97b] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=35.5 K 2024-12-04T14:58:35,152 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43561 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:35,152 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/A is initiating minor compaction (all files) 2024-12-04T14:58:35,152 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/A in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:35,153 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/70e9cf57722045eca89793b53cab3175, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e4cfb410f2a347428b72589277869c0f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/2a48cb6ad136411d879bde76f0f18354] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=42.5 K 2024-12-04T14:58:35,153 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 49e84e9317a9494083825557acbe6f0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733324313559 2024-12-04T14:58:35,157 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70e9cf57722045eca89793b53cab3175, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733324313559 2024-12-04T14:58:35,158 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting db11f686763a4ffc93db4329e7975eeb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733324313674 2024-12-04T14:58:35,159 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4cfb410f2a347428b72589277869c0f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733324313674 2024-12-04T14:58:35,159 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting af1c86342cb14b9d89e4d4c73972a97b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733324314102 2024-12-04T14:58:35,161 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a48cb6ad136411d879bde76f0f18354, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733324314084 2024-12-04T14:58:35,195 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#A#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:35,196 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/62d0a4c31fd648ea90f86da925dde2ac is 50, key is test_row_0/A:col10/1733324314102/Put/seqid=0 2024-12-04T14:58:35,200 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#B#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:35,201 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/c7bbf96895ff4c728067b408813bcc1c is 50, key is test_row_0/B:col10/1733324314102/Put/seqid=0 2024-12-04T14:58:35,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741858_1034 (size=12391) 2024-12-04T14:58:35,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741859_1035 (size=12391) 2024-12-04T14:58:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T14:58:35,251 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:35,251 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:35,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-04T14:58:35,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T14:58:35,256 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:35,257 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:35,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:35,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-04T14:58:35,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:35,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:35,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:35,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:35,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:35,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:35,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/ee9097f1c06c4b7fac0916ae87186d4e is 50, key is test_row_0/A:col10/1733324314616/Put/seqid=0 2024-12-04T14:58:35,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59310 deadline: 1733324375300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:35,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324375302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:35,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324375307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:35,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:35,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324375310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:35,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:35,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:35,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324375317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:35,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741860_1036 (size=12151) 2024-12-04T14:58:35,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:35,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/ee9097f1c06c4b7fac0916ae87186d4e 2024-12-04T14:58:35,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T14:58:35,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/81b28890700e4b96a5f03a5125de73a2 is 50, key is test_row_0/B:col10/1733324314616/Put/seqid=0 2024-12-04T14:58:35,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741861_1037 (size=12151) 2024-12-04T14:58:35,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/81b28890700e4b96a5f03a5125de73a2 2024-12-04T14:58:35,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T14:58:35,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:35,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:35,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:35,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:35,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:35,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/53c6c9aa31864503a812e17da54ed89f is 50, key is test_row_0/C:col10/1733324314616/Put/seqid=0 2024-12-04T14:58:35,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324375436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:35,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324375441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324375441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:35,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:35,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324375442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:35,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:35,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741862_1038 (size=12151) 2024-12-04T14:58:35,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T14:58:35,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:35,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:35,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:35,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:35,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T14:58:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:35,618 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/62d0a4c31fd648ea90f86da925dde2ac as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/62d0a4c31fd648ea90f86da925dde2ac 2024-12-04T14:58:35,630 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/A of 7fc4055584695509eee4923dfb5273a9 into 62d0a4c31fd648ea90f86da925dde2ac(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:35,630 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:35,631 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/A, priority=13, startTime=1733324315147; duration=0sec 2024-12-04T14:58:35,631 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:35,631 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:A 2024-12-04T14:58:35,631 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:35,633 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36391 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:35,633 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/C is initiating minor compaction (all files) 2024-12-04T14:58:35,634 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/C in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:35,634 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/31afd7f434574f1aac5de6af5b7c1016, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/36cea9cefb7044a1a68faa819a0d9149, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/f3fe9f09221e41a58027a23ca71e9211] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=35.5 K 2024-12-04T14:58:35,636 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31afd7f434574f1aac5de6af5b7c1016, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733324313559 2024-12-04T14:58:35,637 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36cea9cefb7044a1a68faa819a0d9149, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733324313674 2024-12-04T14:58:35,638 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3fe9f09221e41a58027a23ca71e9211, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733324314102 2024-12-04T14:58:35,663 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/c7bbf96895ff4c728067b408813bcc1c as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/c7bbf96895ff4c728067b408813bcc1c 2024-12-04T14:58:35,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324375661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:35,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:35,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324375661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:35,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324375662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:35,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:35,675 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#C#compaction#26 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:35,681 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/8d5968f923d346a38222de455c4dd448 is 50, key is test_row_0/C:col10/1733324314102/Put/seqid=0 2024-12-04T14:58:35,682 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/B of 7fc4055584695509eee4923dfb5273a9 into c7bbf96895ff4c728067b408813bcc1c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:35,683 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:35,683 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/B, priority=13, startTime=1733324315148; duration=0sec 2024-12-04T14:58:35,683 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:35,683 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:B 2024-12-04T14:58:35,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324375661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:35,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:35,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T14:58:35,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:35,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:35,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:35,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:35,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:35,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741863_1039 (size=12391) 2024-12-04T14:58:35,798 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/8d5968f923d346a38222de455c4dd448 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/8d5968f923d346a38222de455c4dd448 2024-12-04T14:58:35,817 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/C of 7fc4055584695509eee4923dfb5273a9 into 8d5968f923d346a38222de455c4dd448(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:35,817 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:35,817 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/C, priority=13, startTime=1733324315148; duration=0sec 2024-12-04T14:58:35,817 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:35,817 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:C 2024-12-04T14:58:35,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59310 deadline: 1733324375821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:35,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T14:58:35,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T14:58:35,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:35,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:35,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:35,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:35,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:35,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/53c6c9aa31864503a812e17da54ed89f 2024-12-04T14:58:35,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/ee9097f1c06c4b7fac0916ae87186d4e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/ee9097f1c06c4b7fac0916ae87186d4e 2024-12-04T14:58:35,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/ee9097f1c06c4b7fac0916ae87186d4e, entries=150, sequenceid=190, filesize=11.9 K 2024-12-04T14:58:35,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/81b28890700e4b96a5f03a5125de73a2 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/81b28890700e4b96a5f03a5125de73a2 2024-12-04T14:58:35,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/81b28890700e4b96a5f03a5125de73a2, entries=150, sequenceid=190, filesize=11.9 K 2024-12-04T14:58:35,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/53c6c9aa31864503a812e17da54ed89f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/53c6c9aa31864503a812e17da54ed89f 2024-12-04T14:58:35,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/53c6c9aa31864503a812e17da54ed89f, entries=150, sequenceid=190, filesize=11.9 K 2024-12-04T14:58:35,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 7fc4055584695509eee4923dfb5273a9 in 696ms, sequenceid=190, compaction requested=false 2024-12-04T14:58:35,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:35,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T14:58:35,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:35,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:35,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:35,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:35,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:35,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:35,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/57f212050fcd417fb1b2989b4fc38256 is 50, key is test_row_0/A:col10/1733324315312/Put/seqid=0 2024-12-04T14:58:36,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741864_1040 (size=14541) 2024-12-04T14:58:36,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T14:58:36,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:36,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:36,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:36,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733324316021, max timestamp=1733324316054Num uniques 859; 2024-12-04T14:58:36,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T14:58:36,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733324316021, max timestamp=1733324316054Num uniques 859; 2024-12-04T14:58:36,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T14:58:36,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733324316021, max timestamp=1733324316054Num uniques 859; 2024-12-04T14:58:36,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T14:58:36,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324376144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:36,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324376156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:36,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324376160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:36,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324376162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:36,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:36,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T14:58:36,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:36,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:36,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:36,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324376271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:36,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:36,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324376271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324376271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:36,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324376280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:36,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T14:58:36,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:36,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:36,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:36,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T14:58:36,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/57f212050fcd417fb1b2989b4fc38256 2024-12-04T14:58:36,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324376480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:36,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324376481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324376481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:36,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:36,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/96100a003a754b07af7cd00ea7cbfaa5 is 50, key is test_row_0/B:col10/1733324315312/Put/seqid=0 2024-12-04T14:58:36,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324376510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:36,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:36,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T14:58:36,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:36,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:36,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:36,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741865_1041 (size=12151) 2024-12-04T14:58:36,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T14:58:36,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:36,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:36,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:36,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324376801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:36,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:36,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324376801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:36,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:36,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324376805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:36,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:36,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324376820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:36,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:36,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T14:58:36,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:36,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:36,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:36,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59310 deadline: 1733324376841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:36,852 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 7fc4055584695509eee4923dfb5273a9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:36,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:36,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/96100a003a754b07af7cd00ea7cbfaa5 2024-12-04T14:58:36,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/0acb978d2e7541028508ffaaadc560d2 is 50, key is test_row_0/C:col10/1733324315312/Put/seqid=0 2024-12-04T14:58:36,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T14:58:36,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:36,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:36,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:36,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:37,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741866_1042 (size=12151) 2024-12-04T14:58:37,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/0acb978d2e7541028508ffaaadc560d2 2024-12-04T14:58:37,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/57f212050fcd417fb1b2989b4fc38256 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/57f212050fcd417fb1b2989b4fc38256 2024-12-04T14:58:37,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/57f212050fcd417fb1b2989b4fc38256, entries=200, sequenceid=203, filesize=14.2 K 2024-12-04T14:58:37,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/96100a003a754b07af7cd00ea7cbfaa5 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/96100a003a754b07af7cd00ea7cbfaa5 2024-12-04T14:58:37,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/96100a003a754b07af7cd00ea7cbfaa5, entries=150, sequenceid=203, filesize=11.9 K 2024-12-04T14:58:37,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/0acb978d2e7541028508ffaaadc560d2 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/0acb978d2e7541028508ffaaadc560d2 2024-12-04T14:58:37,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/0acb978d2e7541028508ffaaadc560d2, entries=150, sequenceid=203, filesize=11.9 K 2024-12-04T14:58:37,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 7fc4055584695509eee4923dfb5273a9 in 1134ms, sequenceid=203, compaction requested=true 2024-12-04T14:58:37,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:37,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:37,109 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:37,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:37,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:37,109 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:37,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:37,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:37,111 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39083 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:37,111 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:37,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:37,111 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/A is initiating minor compaction (all files) 2024-12-04T14:58:37,111 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/B is initiating minor compaction (all files) 2024-12-04T14:58:37,111 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/A in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:37,111 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/B in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:37,112 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/62d0a4c31fd648ea90f86da925dde2ac, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/ee9097f1c06c4b7fac0916ae87186d4e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/57f212050fcd417fb1b2989b4fc38256] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=38.2 K 2024-12-04T14:58:37,112 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/c7bbf96895ff4c728067b408813bcc1c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/81b28890700e4b96a5f03a5125de73a2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/96100a003a754b07af7cd00ea7cbfaa5] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=35.8 K 2024-12-04T14:58:37,113 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62d0a4c31fd648ea90f86da925dde2ac, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733324314102 2024-12-04T14:58:37,113 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting c7bbf96895ff4c728067b408813bcc1c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733324314102 2024-12-04T14:58:37,114 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee9097f1c06c4b7fac0916ae87186d4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733324314616 2024-12-04T14:58:37,114 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 81b28890700e4b96a5f03a5125de73a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733324314616 2024-12-04T14:58:37,114 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57f212050fcd417fb1b2989b4fc38256, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733324315293 2024-12-04T14:58:37,114 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 96100a003a754b07af7cd00ea7cbfaa5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733324315301 2024-12-04T14:58:37,145 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#A#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:37,146 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/28a5a0df1c754150aaae6ef4df057fb0 is 50, key is test_row_0/A:col10/1733324315312/Put/seqid=0 2024-12-04T14:58:37,149 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#B#compaction#31 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:37,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T14:58:37,151 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/92adb45f3c2b47d3bdd64b6df1923c5e is 50, key is test_row_0/B:col10/1733324315312/Put/seqid=0 2024-12-04T14:58:37,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:37,151 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-04T14:58:37,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:37,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:37,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:37,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:37,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:37,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:37,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/faa8d40417a042e981a48c1c148816e4 is 50, key is test_row_0/A:col10/1733324316155/Put/seqid=0 2024-12-04T14:58:37,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741867_1043 (size=12493) 2024-12-04T14:58:37,220 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/28a5a0df1c754150aaae6ef4df057fb0 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/28a5a0df1c754150aaae6ef4df057fb0 2024-12-04T14:58:37,230 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/A of 7fc4055584695509eee4923dfb5273a9 into 28a5a0df1c754150aaae6ef4df057fb0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:37,230 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:37,230 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/A, priority=13, startTime=1733324317109; duration=0sec 2024-12-04T14:58:37,230 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:37,230 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:A 2024-12-04T14:58:37,231 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:37,232 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:37,232 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/C is initiating minor compaction (all files) 2024-12-04T14:58:37,232 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/C in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:37,233 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/8d5968f923d346a38222de455c4dd448, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/53c6c9aa31864503a812e17da54ed89f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/0acb978d2e7541028508ffaaadc560d2] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=35.8 K 2024-12-04T14:58:37,233 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d5968f923d346a38222de455c4dd448, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733324314102 2024-12-04T14:58:37,234 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53c6c9aa31864503a812e17da54ed89f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733324314616 2024-12-04T14:58:37,237 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0acb978d2e7541028508ffaaadc560d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733324315301 2024-12-04T14:58:37,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741868_1044 (size=12493) 2024-12-04T14:58:37,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741869_1045 (size=12151) 2024-12-04T14:58:37,249 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/faa8d40417a042e981a48c1c148816e4 2024-12-04T14:58:37,263 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#C#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:37,263 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/d819d1f2d32c40d1ab5fa8c34b79f855 is 50, key is test_row_0/C:col10/1733324315312/Put/seqid=0 2024-12-04T14:58:37,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/21407d91f2ce48d8a46f5ecaeca1b244 is 50, key is test_row_0/B:col10/1733324316155/Put/seqid=0 2024-12-04T14:58:37,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:37,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741870_1046 (size=12493) 2024-12-04T14:58:37,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324377327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:37,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:37,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741871_1047 (size=12151) 2024-12-04T14:58:37,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:37,344 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/21407d91f2ce48d8a46f5ecaeca1b244 2024-12-04T14:58:37,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324377336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:37,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:37,352 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/d819d1f2d32c40d1ab5fa8c34b79f855 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d819d1f2d32c40d1ab5fa8c34b79f855 2024-12-04T14:58:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324377334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:37,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:37,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324377340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:37,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:37,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/ae300b003fc04870a54b2f13e82bb25e is 50, key is test_row_0/C:col10/1733324316155/Put/seqid=0 2024-12-04T14:58:37,369 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/C of 7fc4055584695509eee4923dfb5273a9 into d819d1f2d32c40d1ab5fa8c34b79f855(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:37,369 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:37,369 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/C, priority=13, startTime=1733324317110; duration=0sec 2024-12-04T14:58:37,369 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:37,369 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:C 2024-12-04T14:58:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T14:58:37,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741872_1048 (size=12151) 2024-12-04T14:58:37,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324377440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:37,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324377461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:37,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:37,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:37,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324377470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:37,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:37,657 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/92adb45f3c2b47d3bdd64b6df1923c5e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/92adb45f3c2b47d3bdd64b6df1923c5e 2024-12-04T14:58:37,668 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/B of 7fc4055584695509eee4923dfb5273a9 into 92adb45f3c2b47d3bdd64b6df1923c5e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:37,669 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:37,669 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/B, priority=13, startTime=1733324317109; duration=0sec 2024-12-04T14:58:37,669 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:37,669 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:B 2024-12-04T14:58:37,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:37,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324377661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:37,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:37,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:37,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324377681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324377670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:37,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:37,814 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/ae300b003fc04870a54b2f13e82bb25e 2024-12-04T14:58:37,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/faa8d40417a042e981a48c1c148816e4 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/faa8d40417a042e981a48c1c148816e4 2024-12-04T14:58:37,844 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/faa8d40417a042e981a48c1c148816e4, entries=150, sequenceid=239, filesize=11.9 K 2024-12-04T14:58:37,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/21407d91f2ce48d8a46f5ecaeca1b244 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/21407d91f2ce48d8a46f5ecaeca1b244 2024-12-04T14:58:37,858 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/21407d91f2ce48d8a46f5ecaeca1b244, entries=150, sequenceid=239, filesize=11.9 K 2024-12-04T14:58:37,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/ae300b003fc04870a54b2f13e82bb25e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/ae300b003fc04870a54b2f13e82bb25e 2024-12-04T14:58:37,882 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/ae300b003fc04870a54b2f13e82bb25e, entries=150, sequenceid=239, filesize=11.9 K 2024-12-04T14:58:37,885 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 7fc4055584695509eee4923dfb5273a9 in 733ms, sequenceid=239, compaction requested=false 2024-12-04T14:58:37,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:37,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:37,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-04T14:58:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-04T14:58:37,895 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-04T14:58:37,895 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6300 sec 2024-12-04T14:58:37,909 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.6470 sec 2024-12-04T14:58:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:37,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T14:58:37,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:37,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:37,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:37,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:37,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:37,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:38,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/e38636b25416446b88587d9abafc8208 is 50, key is test_row_0/A:col10/1733324317983/Put/seqid=0 2024-12-04T14:58:38,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741873_1049 (size=14541) 2024-12-04T14:58:38,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733324318036, max timestamp=1733324318100Num uniques 859; 2024-12-04T14:58:38,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733324318036, max timestamp=1733324318100Num uniques 859; 2024-12-04T14:58:38,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T14:58:38,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T14:58:38,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733324318036, max timestamp=1733324318100Num uniques 859; 2024-12-04T14:58:38,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T14:58:38,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:38,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324378175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:38,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324378175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:38,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324378184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:38,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324378291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:38,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:38,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324378291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:38,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:38,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:38,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:38,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324378311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:38,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:38,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:38,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324378371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:38,375 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_1' in 7fc4055584695509eee4923dfb5273a9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:38,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:38,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/e38636b25416446b88587d9abafc8208 2024-12-04T14:58:38,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/e6a1c61d61d842549c4e8dd10b27e952 is 50, key is test_row_0/B:col10/1733324317983/Put/seqid=0 2024-12-04T14:58:38,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741874_1050 (size=12151) 2024-12-04T14:58:38,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/e6a1c61d61d842549c4e8dd10b27e952 2024-12-04T14:58:38,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324378501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324378501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:38,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:38,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324378521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:38,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/0f75f6178a494f67acdf5d85999e411a is 50, key is test_row_0/C:col10/1733324317983/Put/seqid=0 2024-12-04T14:58:38,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741875_1051 (size=12151) 2024-12-04T14:58:38,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324378810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324378811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:38,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324378830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:38,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:38,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:38,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59310 deadline: 1733324378860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:38,865 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 7fc4055584695509eee4923dfb5273a9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:39,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/0f75f6178a494f67acdf5d85999e411a 2024-12-04T14:58:39,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/e38636b25416446b88587d9abafc8208 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e38636b25416446b88587d9abafc8208 2024-12-04T14:58:39,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e38636b25416446b88587d9abafc8208, entries=200, sequenceid=252, filesize=14.2 K 2024-12-04T14:58:39,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/e6a1c61d61d842549c4e8dd10b27e952 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/e6a1c61d61d842549c4e8dd10b27e952 2024-12-04T14:58:39,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/e6a1c61d61d842549c4e8dd10b27e952, entries=150, sequenceid=252, filesize=11.9 K 2024-12-04T14:58:39,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/0f75f6178a494f67acdf5d85999e411a as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/0f75f6178a494f67acdf5d85999e411a 2024-12-04T14:58:39,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/0f75f6178a494f67acdf5d85999e411a, entries=150, sequenceid=252, filesize=11.9 K 2024-12-04T14:58:39,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7fc4055584695509eee4923dfb5273a9 in 1089ms, sequenceid=252, compaction requested=true 2024-12-04T14:58:39,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:39,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:39,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:39,078 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:39,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:39,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:39,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:39,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:39,078 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:39,080 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:39,080 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/B is initiating minor compaction (all files) 2024-12-04T14:58:39,080 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/B in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:39,081 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/92adb45f3c2b47d3bdd64b6df1923c5e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/21407d91f2ce48d8a46f5ecaeca1b244, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/e6a1c61d61d842549c4e8dd10b27e952] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=35.9 K 2024-12-04T14:58:39,081 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:39,081 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/A is initiating minor compaction (all files) 2024-12-04T14:58:39,081 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 92adb45f3c2b47d3bdd64b6df1923c5e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733324315301 2024-12-04T14:58:39,081 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/A in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:39,081 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/28a5a0df1c754150aaae6ef4df057fb0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/faa8d40417a042e981a48c1c148816e4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e38636b25416446b88587d9abafc8208] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=38.3 K 2024-12-04T14:58:39,082 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 21407d91f2ce48d8a46f5ecaeca1b244, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733324316145 2024-12-04T14:58:39,082 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28a5a0df1c754150aaae6ef4df057fb0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733324315301 2024-12-04T14:58:39,083 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting e6a1c61d61d842549c4e8dd10b27e952, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733324317314 2024-12-04T14:58:39,083 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting faa8d40417a042e981a48c1c148816e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733324316145 2024-12-04T14:58:39,084 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e38636b25416446b88587d9abafc8208, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733324317314 2024-12-04T14:58:39,112 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#A#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:39,113 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/6974c2d66eba4971ad19819192771143 is 50, key is test_row_0/A:col10/1733324317983/Put/seqid=0 2024-12-04T14:58:39,123 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#B#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:39,124 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/49dc5e975ce84f3985e2542bb0b4401a is 50, key is test_row_0/B:col10/1733324317983/Put/seqid=0 2024-12-04T14:58:39,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741876_1052 (size=12595) 2024-12-04T14:58:39,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741877_1053 (size=12595) 2024-12-04T14:58:39,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-04T14:58:39,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:39,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:39,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:39,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:39,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:39,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:39,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/f9a024da50714d9489fde75c92668d27 is 50, key is test_row_0/A:col10/1733324319335/Put/seqid=0 2024-12-04T14:58:39,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324379372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:39,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:39,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324379372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:39,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:39,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324379382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:39,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:39,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741878_1054 (size=17181) 2024-12-04T14:58:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T14:58:39,411 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:39,412 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-04T14:58:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T14:58:39,415 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:39,422 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:39,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:39,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324379490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:39,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:39,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324379501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:39,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:39,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324379501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T14:58:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:39,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T14:58:39,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:39,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:39,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:39,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:39,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:39,599 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/6974c2d66eba4971ad19819192771143 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/6974c2d66eba4971ad19819192771143 2024-12-04T14:58:39,613 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/A of 7fc4055584695509eee4923dfb5273a9 into 6974c2d66eba4971ad19819192771143(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:39,613 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/49dc5e975ce84f3985e2542bb0b4401a as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/49dc5e975ce84f3985e2542bb0b4401a 2024-12-04T14:58:39,613 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:39,613 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/A, priority=13, startTime=1733324319078; duration=0sec 2024-12-04T14:58:39,613 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:39,614 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:A 2024-12-04T14:58:39,614 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:39,615 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:39,616 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/C is initiating minor compaction (all files) 2024-12-04T14:58:39,616 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/C in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:39,616 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d819d1f2d32c40d1ab5fa8c34b79f855, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/ae300b003fc04870a54b2f13e82bb25e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/0f75f6178a494f67acdf5d85999e411a] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=35.9 K 2024-12-04T14:58:39,616 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting d819d1f2d32c40d1ab5fa8c34b79f855, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733324315301 2024-12-04T14:58:39,617 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae300b003fc04870a54b2f13e82bb25e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733324316145 2024-12-04T14:58:39,617 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f75f6178a494f67acdf5d85999e411a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733324317314 2024-12-04T14:58:39,622 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/B of 7fc4055584695509eee4923dfb5273a9 into 49dc5e975ce84f3985e2542bb0b4401a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:39,622 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:39,622 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/B, priority=13, startTime=1733324319078; duration=0sec 2024-12-04T14:58:39,622 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:39,622 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:B 2024-12-04T14:58:39,632 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:39,633 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/fe9ccc378f46403d9a9e093bec9e8f5f is 50, key is test_row_0/C:col10/1733324317983/Put/seqid=0 2024-12-04T14:58:39,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741879_1055 (size=12595) 2024-12-04T14:58:39,696 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/fe9ccc378f46403d9a9e093bec9e8f5f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/fe9ccc378f46403d9a9e093bec9e8f5f 2024-12-04T14:58:39,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324379700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:39,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:39,723 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/C of 7fc4055584695509eee4923dfb5273a9 into fe9ccc378f46403d9a9e093bec9e8f5f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:39,723 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:39,723 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/C, priority=13, startTime=1733324319078; duration=0sec 2024-12-04T14:58:39,723 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:39,723 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:C 2024-12-04T14:58:39,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324379723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:39,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T14:58:39,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324379731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:39,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T14:58:39,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:39,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:39,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:39,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:39,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:39,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/f9a024da50714d9489fde75c92668d27 2024-12-04T14:58:39,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/cc625bbb7cd64d06b0a3a5ae23b6e493 is 50, key is test_row_0/B:col10/1733324319335/Put/seqid=0 2024-12-04T14:58:39,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741880_1056 (size=12301) 2024-12-04T14:58:39,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/cc625bbb7cd64d06b0a3a5ae23b6e493 2024-12-04T14:58:39,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T14:58:39,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:39,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:39,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:39,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:39,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:39,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/3641d1c42bfe49c59357130b984e7b99 is 50, key is test_row_0/C:col10/1733324319335/Put/seqid=0 2024-12-04T14:58:39,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741881_1057 (size=12301) 2024-12-04T14:58:40,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324380020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:40,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T14:58:40,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T14:58:40,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:40,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:40,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:40,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:40,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:40,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324380041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:40,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:40,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324380051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:40,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:40,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T14:58:40,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:40,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:40,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:40,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:40,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:40,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/3641d1c42bfe49c59357130b984e7b99 2024-12-04T14:58:40,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T14:58:40,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:40,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:40,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:40,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:40,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:40,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/f9a024da50714d9489fde75c92668d27 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/f9a024da50714d9489fde75c92668d27 2024-12-04T14:58:40,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/f9a024da50714d9489fde75c92668d27, entries=250, sequenceid=289, filesize=16.8 K 2024-12-04T14:58:40,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/cc625bbb7cd64d06b0a3a5ae23b6e493 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/cc625bbb7cd64d06b0a3a5ae23b6e493 2024-12-04T14:58:40,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:40,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324380404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:40,406 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_1' in 7fc4055584695509eee4923dfb5273a9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:40,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:40,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/cc625bbb7cd64d06b0a3a5ae23b6e493, entries=150, sequenceid=289, filesize=12.0 K 2024-12-04T14:58:40,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/3641d1c42bfe49c59357130b984e7b99 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/3641d1c42bfe49c59357130b984e7b99 2024-12-04T14:58:40,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/3641d1c42bfe49c59357130b984e7b99, entries=150, sequenceid=289, filesize=12.0 K 2024-12-04T14:58:40,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=40.25 KB/41220 for 7fc4055584695509eee4923dfb5273a9 in 1107ms, sequenceid=289, compaction requested=false 2024-12-04T14:58:40,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:40,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T14:58:40,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:40,521 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-04T14:58:40,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:40,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:40,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:40,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:40,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/17f2da4642724d9f98879cd72befe71d is 50, key is test_row_0/A:col10/1733324319357/Put/seqid=0 2024-12-04T14:58:40,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:40,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T14:58:40,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741882_1058 (size=12301) 2024-12-04T14:58:40,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733324320578, max timestamp=1733324320601Num uniques 859; 2024-12-04T14:58:40,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T14:58:40,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733324320578, max timestamp=1733324320601Num uniques 859; 2024-12-04T14:58:40,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T14:58:40,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733324320593, max timestamp=1733324320601Num uniques 859; 2024-12-04T14:58:40,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-04T14:58:40,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:40,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324380655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:40,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324380655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:40,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:40,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324380659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:40,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:40,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324380771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:40,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:40,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:40,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324380770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:40,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:40,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324380771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:40,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:40,966 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/17f2da4642724d9f98879cd72befe71d 2024-12-04T14:58:40,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/ddfc9c8ed539438b81d00dcf995f8b6f is 50, key is test_row_0/B:col10/1733324319357/Put/seqid=0 2024-12-04T14:58:40,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:40,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324380990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:40,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324380990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:40,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:41,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:41,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324381001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:41,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:41,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741883_1059 (size=12301) 2024-12-04T14:58:41,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:41,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324381300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324381300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:41,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:41,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:41,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:41,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324381310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:41,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:41,411 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/ddfc9c8ed539438b81d00dcf995f8b6f 2024-12-04T14:58:41,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/a39f37b74372455fbd9f301026abffa7 is 50, key is test_row_0/C:col10/1733324319357/Put/seqid=0 2024-12-04T14:58:41,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741884_1060 (size=12301) 2024-12-04T14:58:41,485 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/a39f37b74372455fbd9f301026abffa7 2024-12-04T14:58:41,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/17f2da4642724d9f98879cd72befe71d as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/17f2da4642724d9f98879cd72befe71d 2024-12-04T14:58:41,504 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/17f2da4642724d9f98879cd72befe71d, entries=150, sequenceid=301, filesize=12.0 K 2024-12-04T14:58:41,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/ddfc9c8ed539438b81d00dcf995f8b6f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/ddfc9c8ed539438b81d00dcf995f8b6f 2024-12-04T14:58:41,519 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/ddfc9c8ed539438b81d00dcf995f8b6f, entries=150, sequenceid=301, filesize=12.0 K 2024-12-04T14:58:41,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/a39f37b74372455fbd9f301026abffa7 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/a39f37b74372455fbd9f301026abffa7 2024-12-04T14:58:41,532 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/a39f37b74372455fbd9f301026abffa7, entries=150, sequenceid=301, filesize=12.0 K 2024-12-04T14:58:41,536 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=142.73 KB/146156 for 7fc4055584695509eee4923dfb5273a9 in 1015ms, sequenceid=301, compaction requested=true 2024-12-04T14:58:41,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:41,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:41,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-04T14:58:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-04T14:58:41,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-04T14:58:41,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1170 sec 2024-12-04T14:58:41,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.1320 sec 2024-12-04T14:58:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T14:58:41,561 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:41,562 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-04T14:58:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T14:58:41,570 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:41,571 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:41,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:41,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T14:58:41,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T14:58:41,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:41,726 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=142.73 KB heapSize=404.38 KB 2024-12-04T14:58:41,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:41,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:41,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:41,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:41,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:41,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:41,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/beaf155425a44e1e9a069a6cb024bdfe is 50, key is test_row_0/A:col10/1733324320648/Put/seqid=0 2024-12-04T14:58:41,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741885_1061 (size=12301) 2024-12-04T14:58:41,775 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/beaf155425a44e1e9a069a6cb024bdfe 2024-12-04T14:58:41,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/6304a6836eaa4666a8853e086ddd5989 is 50, key is test_row_0/B:col10/1733324320648/Put/seqid=0 2024-12-04T14:58:41,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:41,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:41,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324381835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:41,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:41,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324381841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324381841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:41,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741886_1062 (size=12301) 2024-12-04T14:58:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T14:58:41,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:41,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324381950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:41,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:41,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324381950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:41,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:41,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324381950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:41,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:42,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324382163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:42,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:42,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324382163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:42,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:42,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324382163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:42,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T14:58:42,277 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/6304a6836eaa4666a8853e086ddd5989 2024-12-04T14:58:42,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/eded0b01665247a68d81dfd37d7648eb is 50, key is test_row_0/C:col10/1733324320648/Put/seqid=0 2024-12-04T14:58:42,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741887_1063 (size=12301) 2024-12-04T14:58:42,320 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/eded0b01665247a68d81dfd37d7648eb 2024-12-04T14:58:42,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/beaf155425a44e1e9a069a6cb024bdfe as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/beaf155425a44e1e9a069a6cb024bdfe 2024-12-04T14:58:42,351 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/beaf155425a44e1e9a069a6cb024bdfe, entries=150, sequenceid=338, filesize=12.0 K 2024-12-04T14:58:42,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/6304a6836eaa4666a8853e086ddd5989 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/6304a6836eaa4666a8853e086ddd5989 2024-12-04T14:58:42,369 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/6304a6836eaa4666a8853e086ddd5989, entries=150, sequenceid=338, filesize=12.0 K 2024-12-04T14:58:42,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/eded0b01665247a68d81dfd37d7648eb as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/eded0b01665247a68d81dfd37d7648eb 2024-12-04T14:58:42,385 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/eded0b01665247a68d81dfd37d7648eb, entries=150, sequenceid=338, filesize=12.0 K 2024-12-04T14:58:42,387 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~142.73 KB/146156, heapSize ~404.34 KB/414040, currentSize=46.96 KB/48090 for 7fc4055584695509eee4923dfb5273a9 in 661ms, sequenceid=338, compaction requested=true 2024-12-04T14:58:42,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:42,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:42,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-04T14:58:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-04T14:58:42,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-04T14:58:42,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 818 msec 2024-12-04T14:58:42,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 832 msec 2024-12-04T14:58:42,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T14:58:42,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:42,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:42,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:42,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:42,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:42,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:42,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:42,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/2f736fcc69dd467fa960ccbc27beed89 is 50, key is test_row_0/A:col10/1733324321836/Put/seqid=0 2024-12-04T14:58:42,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741888_1064 (size=17181) 2024-12-04T14:58:42,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324382556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:42,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324382559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:42,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:42,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324382569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:42,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:42,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324382681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:42,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324382681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:42,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:42,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324382690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T14:58:42,701 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:42,701 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-04T14:58:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T14:58:42,704 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:42,705 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:42,705 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:42,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T14:58:42,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T14:58:42,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:42,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:42,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:42,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:42,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:42,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:42,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324382891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:42,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:42,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59310 deadline: 1733324382891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:42,895 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 7fc4055584695509eee4923dfb5273a9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8274 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:42,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:42,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:42,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324382900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:42,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:42,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324382900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:42,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:42,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/2f736fcc69dd467fa960ccbc27beed89 2024-12-04T14:58:42,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/4c45b4656a8748f2a04e4ca812ad2812 is 50, key is test_row_0/B:col10/1733324321836/Put/seqid=0 2024-12-04T14:58:43,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741889_1065 (size=12301) 2024-12-04T14:58:43,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T14:58:43,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:43,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:43,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:43,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:43,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T14:58:43,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T14:58:43,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:43,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:43,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:43,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:43,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:43,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324383202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:43,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:43,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:43,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:43,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324383210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:43,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:43,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:43,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:43,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324383220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:43,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:43,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:43,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:43,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T14:58:43,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:43,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:43,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:43,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:43,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:43,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:43,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T14:58:43,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/4c45b4656a8748f2a04e4ca812ad2812 2024-12-04T14:58:43,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/ca7b84ab1dcf4643945f0d0f6df8b40d is 50, key is test_row_0/C:col10/1733324321836/Put/seqid=0 2024-12-04T14:58:43,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741890_1066 (size=12301) 2024-12-04T14:58:43,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T14:58:43,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:43,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:43,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:43,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:43,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:43,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:43,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T14:58:43,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:43,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:43,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:43,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:43,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:43,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324383720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:43,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:43,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:43,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:43,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324383720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:43,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324383730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:43,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T14:58:43,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:43,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:43,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:43,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:43,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T14:58:43,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/ca7b84ab1dcf4643945f0d0f6df8b40d 2024-12-04T14:58:43,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/2f736fcc69dd467fa960ccbc27beed89 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/2f736fcc69dd467fa960ccbc27beed89 2024-12-04T14:58:43,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/2f736fcc69dd467fa960ccbc27beed89, entries=250, sequenceid=350, filesize=16.8 K 2024-12-04T14:58:43,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/4c45b4656a8748f2a04e4ca812ad2812 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/4c45b4656a8748f2a04e4ca812ad2812 2024-12-04T14:58:43,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/4c45b4656a8748f2a04e4ca812ad2812, entries=150, sequenceid=350, filesize=12.0 K 2024-12-04T14:58:43,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/ca7b84ab1dcf4643945f0d0f6df8b40d as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/ca7b84ab1dcf4643945f0d0f6df8b40d 2024-12-04T14:58:43,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/ca7b84ab1dcf4643945f0d0f6df8b40d, entries=150, sequenceid=350, filesize=12.0 K 2024-12-04T14:58:43,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 7fc4055584695509eee4923dfb5273a9 in 1447ms, sequenceid=350, compaction requested=true 2024-12-04T14:58:43,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:43,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:43,923 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T14:58:43,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:43,924 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T14:58:43,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:43,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:43,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:43,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:43,926 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61799 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T14:58:43,926 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/B is initiating minor compaction (all files) 2024-12-04T14:58:43,926 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/B in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:43,927 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/49dc5e975ce84f3985e2542bb0b4401a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/cc625bbb7cd64d06b0a3a5ae23b6e493, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/ddfc9c8ed539438b81d00dcf995f8b6f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/6304a6836eaa4666a8853e086ddd5989, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/4c45b4656a8748f2a04e4ca812ad2812] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=60.4 K 2024-12-04T14:58:43,928 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 71559 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T14:58:43,928 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/A is initiating minor compaction (all files) 2024-12-04T14:58:43,928 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/A in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:43,928 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/6974c2d66eba4971ad19819192771143, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/f9a024da50714d9489fde75c92668d27, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/17f2da4642724d9f98879cd72befe71d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/beaf155425a44e1e9a069a6cb024bdfe, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/2f736fcc69dd467fa960ccbc27beed89] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=69.9 K 2024-12-04T14:58:43,928 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 49dc5e975ce84f3985e2542bb0b4401a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733324317314 2024-12-04T14:58:43,929 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting cc625bbb7cd64d06b0a3a5ae23b6e493, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733324318172 2024-12-04T14:58:43,930 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6974c2d66eba4971ad19819192771143, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733324317314 2024-12-04T14:58:43,930 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting ddfc9c8ed539438b81d00dcf995f8b6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733324319357 2024-12-04T14:58:43,930 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9a024da50714d9489fde75c92668d27, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733324318156 2024-12-04T14:58:43,931 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17f2da4642724d9f98879cd72befe71d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733324319357 2024-12-04T14:58:43,931 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 6304a6836eaa4666a8853e086ddd5989, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733324320639 2024-12-04T14:58:43,931 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting beaf155425a44e1e9a069a6cb024bdfe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733324320639 2024-12-04T14:58:43,931 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c45b4656a8748f2a04e4ca812ad2812, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733324321836 2024-12-04T14:58:43,932 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f736fcc69dd467fa960ccbc27beed89, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733324321833 2024-12-04T14:58:43,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T14:58:43,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:43,941 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T14:58:43,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:43,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:43,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:43,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:43,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:43,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:43,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/c69d1748bc6e42bcb7d8aa8935dea403 is 50, key is test_row_0/A:col10/1733324322556/Put/seqid=0 2024-12-04T14:58:43,963 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:43,964 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/4d2eefc8585248fc8912cbae615acdf8 is 50, key is test_row_0/B:col10/1733324321836/Put/seqid=0 2024-12-04T14:58:44,020 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#A#compaction#56 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:44,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741891_1067 (size=12301) 2024-12-04T14:58:44,024 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/7da7645bc33442d69768cb3595b842c9 is 50, key is test_row_0/A:col10/1733324321836/Put/seqid=0 2024-12-04T14:58:44,024 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/c69d1748bc6e42bcb7d8aa8935dea403 2024-12-04T14:58:44,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741892_1068 (size=12915) 2024-12-04T14:58:44,047 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/4d2eefc8585248fc8912cbae615acdf8 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/4d2eefc8585248fc8912cbae615acdf8 2024-12-04T14:58:44,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/45d092ad9757459bb5411a2c02511cb4 is 50, key is test_row_0/B:col10/1733324322556/Put/seqid=0 2024-12-04T14:58:44,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741893_1069 (size=12915) 2024-12-04T14:58:44,063 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/B of 7fc4055584695509eee4923dfb5273a9 into 4d2eefc8585248fc8912cbae615acdf8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:44,063 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:44,063 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/B, priority=11, startTime=1733324323923; duration=0sec 2024-12-04T14:58:44,063 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:44,063 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:B 2024-12-04T14:58:44,063 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T14:58:44,069 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61799 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T14:58:44,069 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/C is initiating minor compaction (all files) 2024-12-04T14:58:44,069 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/C in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:44,069 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/fe9ccc378f46403d9a9e093bec9e8f5f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/3641d1c42bfe49c59357130b984e7b99, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/a39f37b74372455fbd9f301026abffa7, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/eded0b01665247a68d81dfd37d7648eb, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/ca7b84ab1dcf4643945f0d0f6df8b40d] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=60.4 K 2024-12-04T14:58:44,070 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting fe9ccc378f46403d9a9e093bec9e8f5f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733324317314 2024-12-04T14:58:44,071 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 3641d1c42bfe49c59357130b984e7b99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733324318172 2024-12-04T14:58:44,071 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting a39f37b74372455fbd9f301026abffa7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733324319357 2024-12-04T14:58:44,071 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting eded0b01665247a68d81dfd37d7648eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733324320639 2024-12-04T14:58:44,072 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting ca7b84ab1dcf4643945f0d0f6df8b40d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733324321836 2024-12-04T14:58:44,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741894_1070 (size=12301) 2024-12-04T14:58:44,116 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#C#compaction#58 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:44,117 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/36bd7cd66f4849038ea632780abf486b is 50, key is test_row_0/C:col10/1733324321836/Put/seqid=0 2024-12-04T14:58:44,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741895_1071 (size=12915) 2024-12-04T14:58:44,174 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/36bd7cd66f4849038ea632780abf486b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/36bd7cd66f4849038ea632780abf486b 2024-12-04T14:58:44,185 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/C of 7fc4055584695509eee4923dfb5273a9 into 36bd7cd66f4849038ea632780abf486b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:44,185 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:44,185 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/C, priority=11, startTime=1733324323925; duration=0sec 2024-12-04T14:58:44,185 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:44,185 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:C 2024-12-04T14:58:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:44,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:44,479 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/7da7645bc33442d69768cb3595b842c9 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/7da7645bc33442d69768cb3595b842c9 2024-12-04T14:58:44,496 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/A of 7fc4055584695509eee4923dfb5273a9 into 7da7645bc33442d69768cb3595b842c9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:44,497 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:44,497 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/A, priority=11, startTime=1733324323923; duration=0sec 2024-12-04T14:58:44,497 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:44,498 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:A 2024-12-04T14:58:44,501 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/45d092ad9757459bb5411a2c02511cb4 2024-12-04T14:58:44,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324384498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:44,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:44,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/f1f8a691a77f4fb4b38c410f6ebba1ed is 50, key is test_row_0/C:col10/1733324322556/Put/seqid=0 2024-12-04T14:58:44,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741896_1072 (size=12301) 2024-12-04T14:58:44,577 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/f1f8a691a77f4fb4b38c410f6ebba1ed 2024-12-04T14:58:44,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/c69d1748bc6e42bcb7d8aa8935dea403 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/c69d1748bc6e42bcb7d8aa8935dea403 2024-12-04T14:58:44,615 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/c69d1748bc6e42bcb7d8aa8935dea403, entries=150, sequenceid=370, filesize=12.0 K 2024-12-04T14:58:44,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324384610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:44,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/45d092ad9757459bb5411a2c02511cb4 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/45d092ad9757459bb5411a2c02511cb4 2024-12-04T14:58:44,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:44,641 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/45d092ad9757459bb5411a2c02511cb4, entries=150, sequenceid=370, filesize=12.0 K 2024-12-04T14:58:44,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/f1f8a691a77f4fb4b38c410f6ebba1ed as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/f1f8a691a77f4fb4b38c410f6ebba1ed 2024-12-04T14:58:44,653 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/f1f8a691a77f4fb4b38c410f6ebba1ed, entries=150, sequenceid=370, filesize=12.0 K 2024-12-04T14:58:44,668 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 7fc4055584695509eee4923dfb5273a9 in 727ms, sequenceid=370, compaction requested=false 2024-12-04T14:58:44,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:44,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:44,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-04T14:58:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-04T14:58:44,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-04T14:58:44,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9750 sec 2024-12-04T14:58:44,685 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.9820 sec 2024-12-04T14:58:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:44,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-04T14:58:44,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:44,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:44,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:44,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:44,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:44,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:44,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/b538202e572e4ce78d006a655601cd85 is 50, key is test_row_0/A:col10/1733324324755/Put/seqid=0 2024-12-04T14:58:44,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741897_1073 (size=12301) 2024-12-04T14:58:44,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:44,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324384822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:44,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:44,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:44,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324384824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:44,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:44,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324384831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:44,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324384833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:44,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T14:58:44,861 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:44,863 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:44,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-04T14:58:44,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T14:58:44,866 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:44,867 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:44,867 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:44,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:44,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324384941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:44,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:44,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324384941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:44,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:44,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:44,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324384950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:44,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:44,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T14:58:45,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-04T14:58:45,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:45,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:45,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:45,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324385151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:45,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:45,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324385151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:45,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:45,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324385151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:45,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:45,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:45,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:45,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324385160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:45,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:45,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-04T14:58:45,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:45,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:45,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:45,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T14:58:45,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/b538202e572e4ce78d006a655601cd85 2024-12-04T14:58:45,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/383d4c0a3c9a41b3ae0cc97bfd20deb2 is 50, key is test_row_0/B:col10/1733324324755/Put/seqid=0 2024-12-04T14:58:45,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741898_1074 (size=12301) 2024-12-04T14:58:45,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/383d4c0a3c9a41b3ae0cc97bfd20deb2 2024-12-04T14:58:45,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/191d57e0594740f9b92e06963baf2f9f is 50, key is test_row_0/C:col10/1733324324755/Put/seqid=0 2024-12-04T14:58:45,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-04T14:58:45,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:45,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:45,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:45,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741899_1075 (size=12301) 2024-12-04T14:58:45,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/191d57e0594740f9b92e06963baf2f9f 2024-12-04T14:58:45,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/b538202e572e4ce78d006a655601cd85 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/b538202e572e4ce78d006a655601cd85 2024-12-04T14:58:45,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/b538202e572e4ce78d006a655601cd85, entries=150, sequenceid=387, filesize=12.0 K 2024-12-04T14:58:45,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/383d4c0a3c9a41b3ae0cc97bfd20deb2 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/383d4c0a3c9a41b3ae0cc97bfd20deb2 2024-12-04T14:58:45,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/383d4c0a3c9a41b3ae0cc97bfd20deb2, entries=150, sequenceid=387, filesize=12.0 K 2024-12-04T14:58:45,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/191d57e0594740f9b92e06963baf2f9f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/191d57e0594740f9b92e06963baf2f9f 2024-12-04T14:58:45,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/191d57e0594740f9b92e06963baf2f9f, entries=150, sequenceid=387, filesize=12.0 K 2024-12-04T14:58:45,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 7fc4055584695509eee4923dfb5273a9 in 680ms, sequenceid=387, compaction requested=true 2024-12-04T14:58:45,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:45,437 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:45,439 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:45,439 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/A is initiating minor compaction (all files) 2024-12-04T14:58:45,439 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/A in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:45,439 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/7da7645bc33442d69768cb3595b842c9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/c69d1748bc6e42bcb7d8aa8935dea403, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/b538202e572e4ce78d006a655601cd85] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=36.6 K 2024-12-04T14:58:45,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:45,442 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7da7645bc33442d69768cb3595b842c9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733324321836 2024-12-04T14:58:45,442 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting c69d1748bc6e42bcb7d8aa8935dea403, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733324322544 2024-12-04T14:58:45,443 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting b538202e572e4ce78d006a655601cd85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733324324484 2024-12-04T14:58:45,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:45,446 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:45,448 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:45,448 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/B is initiating minor compaction (all files) 2024-12-04T14:58:45,448 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/B in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:45,448 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/4d2eefc8585248fc8912cbae615acdf8, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/45d092ad9757459bb5411a2c02511cb4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/383d4c0a3c9a41b3ae0cc97bfd20deb2] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=36.6 K 2024-12-04T14:58:45,451 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d2eefc8585248fc8912cbae615acdf8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733324321836 2024-12-04T14:58:45,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:45,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:45,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:45,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:45,453 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 45d092ad9757459bb5411a2c02511cb4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733324322544 2024-12-04T14:58:45,454 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 383d4c0a3c9a41b3ae0cc97bfd20deb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733324324484 2024-12-04T14:58:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:45,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-04T14:58:45,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:45,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:45,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:45,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:45,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:45,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:45,474 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#A#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:45,475 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/c263d186b4dc467db9da36660661d2cc is 50, key is test_row_0/A:col10/1733324324755/Put/seqid=0 2024-12-04T14:58:45,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/e6fb3f8cf5e84854bcf1d3500b9cc0dc is 50, key is test_row_0/A:col10/1733324325465/Put/seqid=0 2024-12-04T14:58:45,483 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#B#compaction#65 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:45,483 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/3fff403b022c45289cd918297c54337e is 50, key is test_row_0/B:col10/1733324324755/Put/seqid=0 2024-12-04T14:58:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T14:58:45,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-04T14:58:45,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:45,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:45,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:45,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:45,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324385503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:45,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:45,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324385505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:45,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:45,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:45,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324385510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:45,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:45,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741901_1077 (size=13017) 2024-12-04T14:58:45,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741900_1076 (size=14741) 2024-12-04T14:58:45,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741902_1078 (size=13017) 2024-12-04T14:58:45,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324385621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:45,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324385621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324385621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:45,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-04T14:58:45,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:45,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:45,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:45,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324385670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:45,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:45,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-04T14:58:45,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:45,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:45,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:45,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324385830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324385830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:45,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:45,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324385830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:45,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:45,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-04T14:58:45,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:45,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:45,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:45,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/e6fb3f8cf5e84854bcf1d3500b9cc0dc 2024-12-04T14:58:45,986 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/c263d186b4dc467db9da36660661d2cc as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/c263d186b4dc467db9da36660661d2cc 2024-12-04T14:58:45,999 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/3fff403b022c45289cd918297c54337e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/3fff403b022c45289cd918297c54337e 2024-12-04T14:58:46,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/ae109fd14792451aacf5f99e7d458972 is 50, key is test_row_0/B:col10/1733324325465/Put/seqid=0 2024-12-04T14:58:46,000 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/A of 7fc4055584695509eee4923dfb5273a9 into c263d186b4dc467db9da36660661d2cc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:46,000 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T14:58:46,000 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/A, priority=13, startTime=1733324325437; duration=0sec 2024-12-04T14:58:46,000 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:46,000 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:A 2024-12-04T14:58:46,000 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:46,003 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:46,003 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/C is initiating minor compaction (all files) 2024-12-04T14:58:46,003 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/C in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:46,004 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/36bd7cd66f4849038ea632780abf486b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/f1f8a691a77f4fb4b38c410f6ebba1ed, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/191d57e0594740f9b92e06963baf2f9f] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=36.6 K 2024-12-04T14:58:46,004 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36bd7cd66f4849038ea632780abf486b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733324321836 2024-12-04T14:58:46,005 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1f8a691a77f4fb4b38c410f6ebba1ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733324322544 2024-12-04T14:58:46,006 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 191d57e0594740f9b92e06963baf2f9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733324324484 2024-12-04T14:58:46,008 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/B of 7fc4055584695509eee4923dfb5273a9 into 3fff403b022c45289cd918297c54337e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:46,008 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:46,008 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/B, priority=13, startTime=1733324325445; duration=0sec 2024-12-04T14:58:46,008 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:46,008 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:B 2024-12-04T14:58:46,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741903_1079 (size=12301) 2024-12-04T14:58:46,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/ae109fd14792451aacf5f99e7d458972 2024-12-04T14:58:46,036 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#C#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:46,037 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/7c0b4da0d4834acdb10785ba0a2c0c2c is 50, key is test_row_0/C:col10/1733324324755/Put/seqid=0 2024-12-04T14:58:46,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/1775f8e86ba84e4baef4ae570f5e2af7 is 50, key is test_row_0/C:col10/1733324325465/Put/seqid=0 2024-12-04T14:58:46,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741904_1080 (size=13017) 2024-12-04T14:58:46,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741905_1081 (size=12301) 2024-12-04T14:58:46,097 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/7c0b4da0d4834acdb10785ba0a2c0c2c as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/7c0b4da0d4834acdb10785ba0a2c0c2c 2024-12-04T14:58:46,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/1775f8e86ba84e4baef4ae570f5e2af7 2024-12-04T14:58:46,105 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/C of 7fc4055584695509eee4923dfb5273a9 into 7c0b4da0d4834acdb10785ba0a2c0c2c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:46,105 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:46,105 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/C, priority=13, startTime=1733324325452; duration=0sec 2024-12-04T14:58:46,106 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:46,106 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:C 2024-12-04T14:58:46,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/e6fb3f8cf5e84854bcf1d3500b9cc0dc as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e6fb3f8cf5e84854bcf1d3500b9cc0dc 2024-12-04T14:58:46,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-04T14:58:46,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:46,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:46,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:46,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:46,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:46,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e6fb3f8cf5e84854bcf1d3500b9cc0dc, entries=200, sequenceid=406, filesize=14.4 K 2024-12-04T14:58:46,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/ae109fd14792451aacf5f99e7d458972 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/ae109fd14792451aacf5f99e7d458972 2024-12-04T14:58:46,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/ae109fd14792451aacf5f99e7d458972, entries=150, sequenceid=406, filesize=12.0 K 2024-12-04T14:58:46,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:46,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/1775f8e86ba84e4baef4ae570f5e2af7 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/1775f8e86ba84e4baef4ae570f5e2af7 2024-12-04T14:58:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324386141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324386141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:46,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324386141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:46,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:46,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/1775f8e86ba84e4baef4ae570f5e2af7, entries=150, sequenceid=406, filesize=12.0 K 2024-12-04T14:58:46,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 7fc4055584695509eee4923dfb5273a9 in 697ms, sequenceid=406, compaction requested=false 2024-12-04T14:58:46,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:46,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-04T14:58:46,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:46,267 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T14:58:46,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:46,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:46,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:46,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:46,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:46,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:46,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/5042a8d5692043d38d9977ad6a4b36ae is 50, key is test_row_0/A:col10/1733324325503/Put/seqid=0 2024-12-04T14:58:46,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741906_1082 (size=12301) 2024-12-04T14:58:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:46,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:46,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324386703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:46,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324386710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:46,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324386713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:46,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324386714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:46,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:46,729 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/5042a8d5692043d38d9977ad6a4b36ae 2024-12-04T14:58:46,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/fc679eba960c45ff846cec0f187f9807 is 50, key is test_row_0/B:col10/1733324325503/Put/seqid=0 2024-12-04T14:58:46,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741907_1083 (size=12301) 2024-12-04T14:58:46,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:46,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324386820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:46,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:46,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:46,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324386820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:46,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324386830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:46,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324386831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:47,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T14:58:47,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:47,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324387031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:47,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:47,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324387031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:47,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:47,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324387040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:47,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:47,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:47,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324387040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:47,189 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/fc679eba960c45ff846cec0f187f9807 2024-12-04T14:58:47,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/6a422d6d86554bf8b9c902b85eee8d55 is 50, key is test_row_0/C:col10/1733324325503/Put/seqid=0 2024-12-04T14:58:47,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741908_1084 (size=12301) 2024-12-04T14:58:47,252 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/6a422d6d86554bf8b9c902b85eee8d55 2024-12-04T14:58:47,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/5042a8d5692043d38d9977ad6a4b36ae as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/5042a8d5692043d38d9977ad6a4b36ae 2024-12-04T14:58:47,279 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/5042a8d5692043d38d9977ad6a4b36ae, entries=150, sequenceid=422, filesize=12.0 K 2024-12-04T14:58:47,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/fc679eba960c45ff846cec0f187f9807 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/fc679eba960c45ff846cec0f187f9807 2024-12-04T14:58:47,306 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/fc679eba960c45ff846cec0f187f9807, entries=150, sequenceid=422, filesize=12.0 K 2024-12-04T14:58:47,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/6a422d6d86554bf8b9c902b85eee8d55 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/6a422d6d86554bf8b9c902b85eee8d55 2024-12-04T14:58:47,331 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/6a422d6d86554bf8b9c902b85eee8d55, entries=150, sequenceid=422, filesize=12.0 K 2024-12-04T14:58:47,333 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 7fc4055584695509eee4923dfb5273a9 in 1066ms, sequenceid=422, compaction requested=true 2024-12-04T14:58:47,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:47,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:47,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-04T14:58:47,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-04T14:58:47,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-04T14:58:47,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4680 sec 2024-12-04T14:58:47,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.4750 sec 2024-12-04T14:58:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:47,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T14:58:47,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324326701, max timestamp=1733324327353Num uniques 859; 2024-12-04T14:58:47,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324326701, max timestamp=1733324327353Num uniques 859; 2024-12-04T14:58:47,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:58:47,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:47,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:58:47,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324326701, max timestamp=1733324327353Num uniques 859; 2024-12-04T14:58:47,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:58:47,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:47,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:47,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:47,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:47,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:47,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/f7bbd19a907a41a59d512a334790127d is 50, key is test_row_0/A:col10/1733324326708/Put/seqid=0 2024-12-04T14:58:47,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324327401, max timestamp=1733324327415Num uniques 859; 2024-12-04T14:58:47,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:58:47,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324327389, max timestamp=1733324327414Num uniques 859; 2024-12-04T14:58:47,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:58:47,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733324327401, max timestamp=1733324327415Num uniques 859; 2024-12-04T14:58:47,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-04T14:58:47,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741909_1085 (size=12301) 2024-12-04T14:58:47,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733324327458, max timestamp=1733324327504Num uniques 1068; 2024-12-04T14:58:47,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-04T14:58:47,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733324327458, max timestamp=1733324327504Num uniques 1009; 2024-12-04T14:58:47,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-04T14:58:47,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733324327458, max timestamp=1733324327504Num uniques 1018; 2024-12-04T14:58:47,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T14:58:47,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733324327535, max timestamp=1733324327552Num uniques 1027; 2024-12-04T14:58:47,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-04T14:58:47,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733324327535, max timestamp=1733324327552Num uniques 1027; 2024-12-04T14:58:47,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-04T14:58:47,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733324327535, max timestamp=1733324327552Num uniques 1027; 2024-12-04T14:58:47,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-04T14:58:47,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1733324327574, max timestamp=1733324327608Num uniques 1036; 2024-12-04T14:58:47,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1733324327574, max timestamp=1733324327608Num uniques 1036; 2024-12-04T14:58:47,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=17676, suffix heap size=135712,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=386 2024-12-04T14:58:47,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=17676, suffix heap size=135712,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=386 2024-12-04T14:58:47,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1733324327574, max timestamp=1733324327608Num uniques 1036; 2024-12-04T14:58:47,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=17676, suffix heap size=135712,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=386 2024-12-04T14:58:47,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733324327634, max timestamp=1733324327657Num uniques 1245; 2024-12-04T14:58:47,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733324327634, max timestamp=1733324327657Num uniques 1245; 2024-12-04T14:58:47,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=57020, new segment data size=13510, suffix heap size=157448,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1245, new segment cells count=295 2024-12-04T14:58:47,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=57020, new segment data size=13510, suffix heap size=157448,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1245, new segment cells count=295 2024-12-04T14:58:47,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13500, totalHeapSize=30920, min timestamp=1733324327634, max timestamp=1733324327657Num uniques 1245; 2024-12-04T14:58:47,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=57010, new segment data size=13500, suffix heap size=157448,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1245, new segment cells count=295 2024-12-04T14:58:47,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733324327694, max timestamp=1733324327750Num uniques 1154; 2024-12-04T14:58:47,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=11630, suffix heap size=147984,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=254 2024-12-04T14:58:47,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733324327694, max timestamp=1733324327750Num uniques 1154; 2024-12-04T14:58:47,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733324327694, max timestamp=1733324327750Num uniques 1154; 2024-12-04T14:58:47,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=11630, suffix heap size=147984,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=254 2024-12-04T14:58:47,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=11630, suffix heap size=147984,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=254 2024-12-04T14:58:47,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=413, cellSize=18908, totalHeapSize=43192, min timestamp=1733324327762, max timestamp=1733324327799Num uniques 1113; 2024-12-04T14:58:47,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=18908, suffix heap size=143720,new segment heap size=43192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=413 2024-12-04T14:58:47,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=413, cellSize=18908, totalHeapSize=43192, min timestamp=1733324327762, max timestamp=1733324327799Num uniques 1113; 2024-12-04T14:58:47,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=18908, suffix heap size=143720,new segment heap size=43192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=413 2024-12-04T14:58:47,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=413, cellSize=18908, totalHeapSize=43192, min timestamp=1733324327762, max timestamp=1733324327799Num uniques 1113; 2024-12-04T14:58:47,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=18908, suffix heap size=143720,new segment heap size=43192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=413 2024-12-04T14:58:47,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/f7bbd19a907a41a59d512a334790127d 2024-12-04T14:58:47,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1733324327821, max timestamp=1733324327849Num uniques 1272; 2024-12-04T14:58:47,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=58252, new segment data size=14742, suffix heap size=160256,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1272, new segment cells count=322 2024-12-04T14:58:47,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1733324327821, max timestamp=1733324327849Num uniques 1272; 2024-12-04T14:58:47,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=58252, new segment data size=14742, suffix heap size=160256,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1272, new segment cells count=322 2024-12-04T14:58:47,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1733324327821, max timestamp=1733324327849Num uniques 1272; 2024-12-04T14:58:47,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=58252, new segment data size=14742, suffix heap size=160256,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1272, new segment cells count=322 2024-12-04T14:58:47,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/3a85cf6dd0424e4980e5c2553371725a is 50, key is test_row_0/B:col10/1733324326708/Put/seqid=0 2024-12-04T14:58:47,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741910_1086 (size=12301) 2024-12-04T14:58:47,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/3a85cf6dd0424e4980e5c2553371725a 2024-12-04T14:58:47,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/1bd2fc57d97c46c6adbe641bf4c7537c is 50, key is test_row_0/C:col10/1733324326708/Put/seqid=0 2024-12-04T14:58:47,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741911_1087 (size=12301) 2024-12-04T14:58:47,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/1bd2fc57d97c46c6adbe641bf4c7537c 2024-12-04T14:58:47,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/f7bbd19a907a41a59d512a334790127d as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/f7bbd19a907a41a59d512a334790127d 2024-12-04T14:58:47,932 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733324327865, max timestamp=1733324327895Num uniques 1181; 2024-12-04T14:58:47,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=8286, suffix heap size=150792,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=181 2024-12-04T14:58:47,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733324327865, max timestamp=1733324327895Num uniques 1181; 2024-12-04T14:58:47,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=6870, suffix heap size=150792,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=150 2024-12-04T14:58:47,932 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733324327865, max timestamp=1733324327895Num uniques 1181; 2024-12-04T14:58:47,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=8286, suffix heap size=150792,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=181 2024-12-04T14:58:47,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/f7bbd19a907a41a59d512a334790127d, entries=150, sequenceid=443, filesize=12.0 K 2024-12-04T14:58:47,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/3a85cf6dd0424e4980e5c2553371725a as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/3a85cf6dd0424e4980e5c2553371725a 2024-12-04T14:58:47,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/3a85cf6dd0424e4980e5c2553371725a, entries=150, sequenceid=443, filesize=12.0 K 2024-12-04T14:58:47,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/1bd2fc57d97c46c6adbe641bf4c7537c as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/1bd2fc57d97c46c6adbe641bf4c7537c 2024-12-04T14:58:47,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/1bd2fc57d97c46c6adbe641bf4c7537c, entries=150, sequenceid=443, filesize=12.0 K 2024-12-04T14:58:47,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=32.16 KB/32934 for 7fc4055584695509eee4923dfb5273a9 in 626ms, sequenceid=443, compaction requested=true 2024-12-04T14:58:47,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:47,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:47,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:47,983 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:58:47,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:47,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:47,983 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:58:47,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:47,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:47,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=52.29 KB heapSize=140.71 KB 2024-12-04T14:58:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:47,986 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:58:47,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:47,986 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/A is initiating minor compaction (all files) 2024-12-04T14:58:47,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:47,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:47,987 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/A in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:47,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:47,987 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:58:47,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:47,987 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/c263d186b4dc467db9da36660661d2cc, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e6fb3f8cf5e84854bcf1d3500b9cc0dc, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/5042a8d5692043d38d9977ad6a4b36ae, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/f7bbd19a907a41a59d512a334790127d] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=51.1 K 2024-12-04T14:58:47,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:47,987 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/B is initiating minor compaction (all files) 2024-12-04T14:58:47,987 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/B in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:47,987 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/3fff403b022c45289cd918297c54337e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/ae109fd14792451aacf5f99e7d458972, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/fc679eba960c45ff846cec0f187f9807, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/3a85cf6dd0424e4980e5c2553371725a] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=48.8 K 2024-12-04T14:58:47,987 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting c263d186b4dc467db9da36660661d2cc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733324324484 2024-12-04T14:58:47,988 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fff403b022c45289cd918297c54337e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733324324484 2024-12-04T14:58:47,988 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6fb3f8cf5e84854bcf1d3500b9cc0dc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733324324820 2024-12-04T14:58:47,988 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting ae109fd14792451aacf5f99e7d458972, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733324324821 2024-12-04T14:58:47,988 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5042a8d5692043d38d9977ad6a4b36ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733324325501 2024-12-04T14:58:47,988 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting fc679eba960c45ff846cec0f187f9807, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733324325501 2024-12-04T14:58:47,989 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a85cf6dd0424e4980e5c2553371725a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733324326708 2024-12-04T14:58:47,990 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7bbd19a907a41a59d512a334790127d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733324326708 2024-12-04T14:58:48,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/2f52fe8b6cc8488a8e97ede8f19e200f is 50, key is test_row_0/A:col10/1733324327983/Put/seqid=0 2024-12-04T14:58:48,025 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#A#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:48,026 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/6ea8b61e25b143279fb864f544e6efb9 is 50, key is test_row_0/A:col10/1733324326708/Put/seqid=0 2024-12-04T14:58:48,035 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#B#compaction#77 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:48,036 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/1c76d61609fa416ca47850d6e81553de is 50, key is test_row_0/B:col10/1733324326708/Put/seqid=0 2024-12-04T14:58:48,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741912_1088 (size=12301) 2024-12-04T14:58:48,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/2f52fe8b6cc8488a8e97ede8f19e200f 2024-12-04T14:58:48,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324328012, max timestamp=1733324328067Num uniques 859; 2024-12-04T14:58:48,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:58:48,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324328012, max timestamp=1733324328067Num uniques 859; 2024-12-04T14:58:48,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:58:48,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324328012, max timestamp=1733324328067Num uniques 859; 2024-12-04T14:58:48,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:58:48,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741913_1089 (size=13153) 2024-12-04T14:58:48,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/c23e65755bb2430886a77aa9d4261490 is 50, key is test_row_0/B:col10/1733324327983/Put/seqid=0 2024-12-04T14:58:48,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741914_1090 (size=13153) 2024-12-04T14:58:48,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324388154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:48,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324388158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:48,169 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/1c76d61609fa416ca47850d6e81553de as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/1c76d61609fa416ca47850d6e81553de 2024-12-04T14:58:48,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:48,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324388160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:48,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:48,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324388167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:48,189 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/B of 7fc4055584695509eee4923dfb5273a9 into 1c76d61609fa416ca47850d6e81553de(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:48,189 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:48,190 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/B, priority=12, startTime=1733324327983; duration=0sec 2024-12-04T14:58:48,190 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:48,190 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:B 2024-12-04T14:58:48,190 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:58:48,192 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:58:48,192 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/C is initiating minor compaction (all files) 2024-12-04T14:58:48,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741915_1091 (size=12301) 2024-12-04T14:58:48,193 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/C in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:48,193 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/7c0b4da0d4834acdb10785ba0a2c0c2c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/1775f8e86ba84e4baef4ae570f5e2af7, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/6a422d6d86554bf8b9c902b85eee8d55, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/1bd2fc57d97c46c6adbe641bf4c7537c] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=48.8 K 2024-12-04T14:58:48,193 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c0b4da0d4834acdb10785ba0a2c0c2c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733324324484 2024-12-04T14:58:48,194 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1775f8e86ba84e4baef4ae570f5e2af7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733324324821 2024-12-04T14:58:48,194 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a422d6d86554bf8b9c902b85eee8d55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733324325501 2024-12-04T14:58:48,194 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bd2fc57d97c46c6adbe641bf4c7537c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733324326708 2024-12-04T14:58:48,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/c23e65755bb2430886a77aa9d4261490 2024-12-04T14:58:48,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/cb845609f9ba4ac68d8b06c4293e8470 is 50, key is test_row_0/C:col10/1733324327983/Put/seqid=0 2024-12-04T14:58:48,213 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#C#compaction#80 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:48,244 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/0d7bdf3ec10948a6a674e85f38cf1422 is 50, key is test_row_0/C:col10/1733324326708/Put/seqid=0 2024-12-04T14:58:48,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324388270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:48,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:48,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324388280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:48,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324388280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:48,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324388280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:48,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741916_1092 (size=12301) 2024-12-04T14:58:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:48,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:48,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741917_1093 (size=13153) 2024-12-04T14:58:48,313 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/0d7bdf3ec10948a6a674e85f38cf1422 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/0d7bdf3ec10948a6a674e85f38cf1422 2024-12-04T14:58:48,322 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/C of 7fc4055584695509eee4923dfb5273a9 into 0d7bdf3ec10948a6a674e85f38cf1422(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:48,323 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:48,323 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/C, priority=12, startTime=1733324327983; duration=0sec 2024-12-04T14:58:48,323 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:48,323 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:C 2024-12-04T14:58:48,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324388480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:48,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324388500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:48,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:48,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324388500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324388500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:48,529 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/6ea8b61e25b143279fb864f544e6efb9 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/6ea8b61e25b143279fb864f544e6efb9 2024-12-04T14:58:48,577 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/A of 7fc4055584695509eee4923dfb5273a9 into 6ea8b61e25b143279fb864f544e6efb9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:48,577 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:48,577 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/A, priority=12, startTime=1733324327983; duration=0sec 2024-12-04T14:58:48,578 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:48,578 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:A 2024-12-04T14:58:48,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.74 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/cb845609f9ba4ac68d8b06c4293e8470 2024-12-04T14:58:48,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/2f52fe8b6cc8488a8e97ede8f19e200f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/2f52fe8b6cc8488a8e97ede8f19e200f 2024-12-04T14:58:48,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/2f52fe8b6cc8488a8e97ede8f19e200f, entries=150, sequenceid=606, filesize=12.0 K 2024-12-04T14:58:48,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/c23e65755bb2430886a77aa9d4261490 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/c23e65755bb2430886a77aa9d4261490 2024-12-04T14:58:48,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/c23e65755bb2430886a77aa9d4261490, entries=150, sequenceid=606, filesize=12.0 K 2024-12-04T14:58:48,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/cb845609f9ba4ac68d8b06c4293e8470 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/cb845609f9ba4ac68d8b06c4293e8470 2024-12-04T14:58:48,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/cb845609f9ba4ac68d8b06c4293e8470, entries=150, sequenceid=606, filesize=12.0 K 2024-12-04T14:58:48,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~59.00 KB/60414, heapSize ~160.59 KB/164440, currentSize=127.47 KB/130530 for 7fc4055584695509eee4923dfb5273a9 in 757ms, sequenceid=606, compaction requested=false 2024-12-04T14:58:48,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:48,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-04T14:58:48,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:48,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:48,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:48,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:48,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:48,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:48,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/981d5788d67b415a80d12585c8adce4b is 50, key is test_row_0/A:col10/1733324328159/Put/seqid=0 2024-12-04T14:58:48,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324388832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:48,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:48,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324388832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:48,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:48,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324388836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:48,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:48,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324388839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:48,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:48,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741918_1094 (size=14741) 2024-12-04T14:58:48,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324388950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:48,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324388950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:48,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324388950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:48,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:48,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324388960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:49,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T14:58:49,042 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:49,042 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-04T14:58:49,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T14:58:49,046 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:49,046 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:49,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:49,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T14:58:49,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:49,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324389161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:49,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:49,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:49,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324389161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:49,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:49,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324389161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:49,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:49,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:49,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324389170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:49,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:49,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-04T14:58:49,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:49,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:49,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:49,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=645 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/981d5788d67b415a80d12585c8adce4b 2024-12-04T14:58:49,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/404a9727bbd744fa9917e51f59851b3d is 50, key is test_row_0/B:col10/1733324328159/Put/seqid=0 2024-12-04T14:58:49,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741919_1095 (size=12301) 2024-12-04T14:58:49,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-04T14:58:49,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:49,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:49,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:49,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T14:58:49,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324389470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:49,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324389470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:49,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:49,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324389480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324389481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:49,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-04T14:58:49,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:49,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:49,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:49,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-04T14:58:49,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:49,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:49,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:49,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T14:58:49,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=645 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/404a9727bbd744fa9917e51f59851b3d 2024-12-04T14:58:49,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/d7ed5645606d4b1ebfa6d51a17b4dcc2 is 50, key is test_row_0/C:col10/1733324328159/Put/seqid=0 2024-12-04T14:58:49,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741920_1096 (size=12301) 2024-12-04T14:58:49,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-04T14:58:49,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:49,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:49,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:49,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-04T14:58:49,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:49,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:49,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:49,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:49,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59336 deadline: 1733324389980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:49,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:49,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324389981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:49,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:49,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:49,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59328 deadline: 1733324389990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:49,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:49,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59306 deadline: 1733324389990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:49,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:50,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-04T14:58:50,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:50,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. as already flushing 2024-12-04T14:58:50,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:50,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:50,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:50,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:50,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T14:58:50,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=645 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/d7ed5645606d4b1ebfa6d51a17b4dcc2 2024-12-04T14:58:50,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/981d5788d67b415a80d12585c8adce4b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/981d5788d67b415a80d12585c8adce4b 2024-12-04T14:58:50,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/981d5788d67b415a80d12585c8adce4b, entries=200, sequenceid=645, filesize=14.4 K 2024-12-04T14:58:50,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/404a9727bbd744fa9917e51f59851b3d as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/404a9727bbd744fa9917e51f59851b3d 2024-12-04T14:58:50,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/404a9727bbd744fa9917e51f59851b3d, entries=150, sequenceid=645, filesize=12.0 K 2024-12-04T14:58:50,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/d7ed5645606d4b1ebfa6d51a17b4dcc2 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d7ed5645606d4b1ebfa6d51a17b4dcc2 2024-12-04T14:58:50,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d7ed5645606d4b1ebfa6d51a17b4dcc2, entries=150, sequenceid=645, filesize=12.0 K 2024-12-04T14:58:50,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 7fc4055584695509eee4923dfb5273a9 in 1446ms, sequenceid=645, compaction requested=true 2024-12-04T14:58:50,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:50,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:50,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:50,255 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:50,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:50,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:50,255 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:50,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:50,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:50,267 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:50,267 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/B is initiating minor compaction (all files) 2024-12-04T14:58:50,267 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/B in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:50,267 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/1c76d61609fa416ca47850d6e81553de, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/c23e65755bb2430886a77aa9d4261490, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/404a9727bbd744fa9917e51f59851b3d] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=36.9 K 2024-12-04T14:58:50,268 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:50,268 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c76d61609fa416ca47850d6e81553de, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733324326708 2024-12-04T14:58:50,268 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/A is initiating minor compaction (all files) 2024-12-04T14:58:50,268 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/A in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:50,268 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/6ea8b61e25b143279fb864f544e6efb9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/2f52fe8b6cc8488a8e97ede8f19e200f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/981d5788d67b415a80d12585c8adce4b] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=39.3 K 2024-12-04T14:58:50,269 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ea8b61e25b143279fb864f544e6efb9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733324326708 2024-12-04T14:58:50,270 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting c23e65755bb2430886a77aa9d4261490, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=606, earliestPutTs=1733324327895 2024-12-04T14:58:50,270 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f52fe8b6cc8488a8e97ede8f19e200f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=606, earliestPutTs=1733324327895 2024-12-04T14:58:50,270 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 981d5788d67b415a80d12585c8adce4b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=645, earliestPutTs=1733324328123 2024-12-04T14:58:50,270 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 404a9727bbd744fa9917e51f59851b3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=645, earliestPutTs=1733324328156 2024-12-04T14:58:50,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-04T14:58:50,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:50,279 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T14:58:50,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:50,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:50,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:50,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:50,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:50,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:50,302 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#B#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:50,303 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/e7d4d172081343e698127544e4cb4aa4 is 50, key is test_row_0/B:col10/1733324328159/Put/seqid=0 2024-12-04T14:58:50,307 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#A#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:50,308 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/896d3e2351d54f838b4c49d2e41b5158 is 50, key is test_row_0/A:col10/1733324328159/Put/seqid=0 2024-12-04T14:58:50,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/3ac0cf51043f494a9842b3254198c910 is 50, key is test_row_0/A:col10/1733324328824/Put/seqid=0 2024-12-04T14:58:50,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741921_1097 (size=13255) 2024-12-04T14:58:50,346 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/e7d4d172081343e698127544e4cb4aa4 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/e7d4d172081343e698127544e4cb4aa4 2024-12-04T14:58:50,357 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/B of 7fc4055584695509eee4923dfb5273a9 into e7d4d172081343e698127544e4cb4aa4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:50,357 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:50,357 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/B, priority=13, startTime=1733324330255; duration=0sec 2024-12-04T14:58:50,357 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:50,357 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:B 2024-12-04T14:58:50,357 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:50,360 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:50,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741923_1099 (size=12301) 2024-12-04T14:58:50,360 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/C is initiating minor compaction (all files) 2024-12-04T14:58:50,360 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/C in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:50,360 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/0d7bdf3ec10948a6a674e85f38cf1422, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/cb845609f9ba4ac68d8b06c4293e8470, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d7ed5645606d4b1ebfa6d51a17b4dcc2] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=36.9 K 2024-12-04T14:58:50,361 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d7bdf3ec10948a6a674e85f38cf1422, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733324326708 2024-12-04T14:58:50,362 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting cb845609f9ba4ac68d8b06c4293e8470, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=606, earliestPutTs=1733324327895 2024-12-04T14:58:50,363 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting d7ed5645606d4b1ebfa6d51a17b4dcc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=645, earliestPutTs=1733324328156 2024-12-04T14:58:50,363 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=655 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/3ac0cf51043f494a9842b3254198c910 2024-12-04T14:58:50,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741922_1098 (size=13255) 2024-12-04T14:58:50,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/29f4fcb505c243068ba3764fe9f589c9 is 50, key is test_row_0/B:col10/1733324328824/Put/seqid=0 2024-12-04T14:58:50,396 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#C#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:50,397 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/48c44b837fe6436d9e586979f994f9af is 50, key is test_row_0/C:col10/1733324328159/Put/seqid=0 2024-12-04T14:58:50,397 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/896d3e2351d54f838b4c49d2e41b5158 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/896d3e2351d54f838b4c49d2e41b5158 2024-12-04T14:58:50,409 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/A of 7fc4055584695509eee4923dfb5273a9 into 896d3e2351d54f838b4c49d2e41b5158(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:50,409 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:50,409 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/A, priority=13, startTime=1733324330255; duration=0sec 2024-12-04T14:58:50,409 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:50,409 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:A 2024-12-04T14:58:50,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741925_1101 (size=13255) 2024-12-04T14:58:50,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741924_1100 (size=12301) 2024-12-04T14:58:50,443 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/48c44b837fe6436d9e586979f994f9af as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/48c44b837fe6436d9e586979f994f9af 2024-12-04T14:58:50,446 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=655 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/29f4fcb505c243068ba3764fe9f589c9 2024-12-04T14:58:50,455 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/C of 7fc4055584695509eee4923dfb5273a9 into 48c44b837fe6436d9e586979f994f9af(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:50,455 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:50,455 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/C, priority=13, startTime=1733324330255; duration=0sec 2024-12-04T14:58:50,455 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:50,455 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:C 2024-12-04T14:58:50,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/d7eaaa75326e4699a0bc4d0f7f987d6d is 50, key is test_row_0/C:col10/1733324328824/Put/seqid=0 2024-12-04T14:58:50,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741926_1102 (size=12301) 2024-12-04T14:58:50,893 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=655 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/d7eaaa75326e4699a0bc4d0f7f987d6d 2024-12-04T14:58:50,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/3ac0cf51043f494a9842b3254198c910 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/3ac0cf51043f494a9842b3254198c910 2024-12-04T14:58:50,911 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/3ac0cf51043f494a9842b3254198c910, entries=150, sequenceid=655, filesize=12.0 K 2024-12-04T14:58:50,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/29f4fcb505c243068ba3764fe9f589c9 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/29f4fcb505c243068ba3764fe9f589c9 2024-12-04T14:58:50,929 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/29f4fcb505c243068ba3764fe9f589c9, entries=150, sequenceid=655, filesize=12.0 K 2024-12-04T14:58:50,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/d7eaaa75326e4699a0bc4d0f7f987d6d as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d7eaaa75326e4699a0bc4d0f7f987d6d 2024-12-04T14:58:50,946 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d7eaaa75326e4699a0bc4d0f7f987d6d, entries=150, sequenceid=655, filesize=12.0 K 2024-12-04T14:58:50,949 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 7fc4055584695509eee4923dfb5273a9 in 670ms, sequenceid=655, compaction requested=false 2024-12-04T14:58:50,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:50,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:50,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-04T14:58:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-04T14:58:50,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-04T14:58:50,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9040 sec 2024-12-04T14:58:50,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.9130 sec 2024-12-04T14:58:51,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T14:58:51,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:51,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:51,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:51,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:51,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:51,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:51,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/15bafea960994934a434b4bef35980d4 is 50, key is test_row_0/A:col10/1733324331004/Put/seqid=0 2024-12-04T14:58:51,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741927_1103 (size=14741) 2024-12-04T14:58:51,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733324331037, max timestamp=1733324331055Num uniques 859; 2024-12-04T14:58:51,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T14:58:51,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733324331037, max timestamp=1733324331055Num uniques 859; 2024-12-04T14:58:51,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T14:58:51,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324331052, max timestamp=1733324331055Num uniques 859; 2024-12-04T14:58:51,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:58:51,093 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-04T14:58:51,093 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:51,098 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-04T14:58:51,098 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:51,099 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:51,099 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:51,099 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:51,100 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:51,100 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-04T14:58:51,100 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:51,100 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:51,100 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:51,100 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:51,101 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:51,101 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:51,107 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-04T14:58:51,107 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:51,107 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:51,107 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:51,107 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:51,108 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-04T14:58:51,108 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:51,108 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:51,108 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:51,109 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:51,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324391091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:51,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:51,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:51,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:51,113 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-04T14:58:51,113 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:51,113 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:51,113 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:51,113 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:51,115 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-04T14:58:51,115 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:51,115 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:51,115 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:51,116 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T14:58:51,191 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:51,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324391220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:51,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324391430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:51,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:51,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:51,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:51,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=668 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/15bafea960994934a434b4bef35980d4 2024-12-04T14:58:51,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/956bbf97a5f14243853de91234a07dd8 is 50, key is test_row_0/B:col10/1733324331004/Put/seqid=0 2024-12-04T14:58:51,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741928_1104 (size=12301) 2024-12-04T14:58:51,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324391740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:51,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=668 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/956bbf97a5f14243853de91234a07dd8 2024-12-04T14:58:51,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/d2a2013868fa4aacb09e00b349a2e304 is 50, key is test_row_0/C:col10/1733324331004/Put/seqid=0 2024-12-04T14:58:51,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741929_1105 (size=12301) 2024-12-04T14:58:52,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59348 deadline: 1733324392251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:52,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=668 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/d2a2013868fa4aacb09e00b349a2e304 2024-12-04T14:58:52,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/15bafea960994934a434b4bef35980d4 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/15bafea960994934a434b4bef35980d4 2024-12-04T14:58:52,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/15bafea960994934a434b4bef35980d4, entries=200, sequenceid=668, filesize=14.4 K 2024-12-04T14:58:52,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/956bbf97a5f14243853de91234a07dd8 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/956bbf97a5f14243853de91234a07dd8 2024-12-04T14:58:52,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/956bbf97a5f14243853de91234a07dd8, entries=150, sequenceid=668, filesize=12.0 K 2024-12-04T14:58:52,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/d2a2013868fa4aacb09e00b349a2e304 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d2a2013868fa4aacb09e00b349a2e304 2024-12-04T14:58:52,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d2a2013868fa4aacb09e00b349a2e304, entries=150, sequenceid=668, filesize=12.0 K 2024-12-04T14:58:52,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7fc4055584695509eee4923dfb5273a9 in 1298ms, sequenceid=668, compaction requested=true 2024-12-04T14:58:52,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:52,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:52,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:52,308 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:52,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:52,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:52,308 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:52,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fc4055584695509eee4923dfb5273a9:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:52,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:52,311 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:52,311 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:52,311 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/B is initiating minor compaction (all files) 2024-12-04T14:58:52,311 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/A is initiating minor compaction (all files) 2024-12-04T14:58:52,311 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/B in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:52,311 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/A in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:52,311 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/e7d4d172081343e698127544e4cb4aa4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/29f4fcb505c243068ba3764fe9f589c9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/956bbf97a5f14243853de91234a07dd8] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=37.0 K 2024-12-04T14:58:52,311 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/896d3e2351d54f838b4c49d2e41b5158, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/3ac0cf51043f494a9842b3254198c910, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/15bafea960994934a434b4bef35980d4] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=39.4 K 2024-12-04T14:58:52,311 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting e7d4d172081343e698127544e4cb4aa4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=645, earliestPutTs=1733324328156 2024-12-04T14:58:52,311 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 896d3e2351d54f838b4c49d2e41b5158, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=645, earliestPutTs=1733324328156 2024-12-04T14:58:52,312 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 29f4fcb505c243068ba3764fe9f589c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=655, earliestPutTs=1733324328824 2024-12-04T14:58:52,312 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ac0cf51043f494a9842b3254198c910, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=655, earliestPutTs=1733324328824 2024-12-04T14:58:52,312 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 956bbf97a5f14243853de91234a07dd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=668, earliestPutTs=1733324330998 2024-12-04T14:58:52,313 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15bafea960994934a434b4bef35980d4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=668, earliestPutTs=1733324330998 2024-12-04T14:58:52,323 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#B#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:52,323 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#A#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:52,323 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/6658ba3b708d45c08a9ab50eb1c717c3 is 50, key is test_row_0/B:col10/1733324331004/Put/seqid=0 2024-12-04T14:58:52,324 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/989fe5384b0b4ffeb00f39d63a395e24 is 50, key is test_row_0/A:col10/1733324331004/Put/seqid=0 2024-12-04T14:58:52,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741930_1106 (size=13357) 2024-12-04T14:58:52,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741931_1107 (size=13357) 2024-12-04T14:58:52,744 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/989fe5384b0b4ffeb00f39d63a395e24 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/989fe5384b0b4ffeb00f39d63a395e24 2024-12-04T14:58:52,744 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/6658ba3b708d45c08a9ab50eb1c717c3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/6658ba3b708d45c08a9ab50eb1c717c3 2024-12-04T14:58:52,749 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/A of 7fc4055584695509eee4923dfb5273a9 into 989fe5384b0b4ffeb00f39d63a395e24(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:52,749 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/B of 7fc4055584695509eee4923dfb5273a9 into 6658ba3b708d45c08a9ab50eb1c717c3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:52,749 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:52,749 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:52,749 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/B, priority=13, startTime=1733324332308; duration=0sec 2024-12-04T14:58:52,749 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/A, priority=13, startTime=1733324332308; duration=0sec 2024-12-04T14:58:52,750 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:52,750 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:B 2024-12-04T14:58:52,750 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:52,750 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:A 2024-12-04T14:58:52,750 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:52,751 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:52,751 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 7fc4055584695509eee4923dfb5273a9/C is initiating minor compaction (all files) 2024-12-04T14:58:52,751 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fc4055584695509eee4923dfb5273a9/C in TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:52,751 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/48c44b837fe6436d9e586979f994f9af, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d7eaaa75326e4699a0bc4d0f7f987d6d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d2a2013868fa4aacb09e00b349a2e304] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp, totalSize=37.0 K 2024-12-04T14:58:52,751 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 48c44b837fe6436d9e586979f994f9af, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=645, earliestPutTs=1733324328156 2024-12-04T14:58:52,752 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting d7eaaa75326e4699a0bc4d0f7f987d6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=655, earliestPutTs=1733324328824 2024-12-04T14:58:52,752 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting d2a2013868fa4aacb09e00b349a2e304, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=668, earliestPutTs=1733324330998 2024-12-04T14:58:52,759 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fc4055584695509eee4923dfb5273a9#C#compaction#95 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:52,760 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/c467797204c94b62aa9ee10ad569d4a9 is 50, key is test_row_0/C:col10/1733324331004/Put/seqid=0 2024-12-04T14:58:52,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741932_1108 (size=13357) 2024-12-04T14:58:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:52,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-04T14:58:52,955 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-04T14:58:52,955 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:52,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:52,955 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:52,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:52,955 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:52,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:52,955 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:52,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:52,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:52,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:58:52,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/07269b59590b4faabf6fd3c467f9681f is 50, key is test_row_0/A:col10/1733324331077/Put/seqid=0 2024-12-04T14:58:52,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741933_1109 (size=12301) 2024-12-04T14:58:53,176 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/c467797204c94b62aa9ee10ad569d4a9 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/c467797204c94b62aa9ee10ad569d4a9 2024-12-04T14:58:53,183 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fc4055584695509eee4923dfb5273a9/C of 7fc4055584695509eee4923dfb5273a9 into c467797204c94b62aa9ee10ad569d4a9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:53,183 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:53,184 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., storeName=7fc4055584695509eee4923dfb5273a9/C, priority=13, startTime=1733324332308; duration=0sec 2024-12-04T14:58:53,184 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:53,184 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fc4055584695509eee4923dfb5273a9:C 2024-12-04T14:58:53,225 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T14:58:53,263 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-04T14:58:53,263 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:53,264 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:53,264 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:53,264 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:53,264 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T14:58:53,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-12-04T14:58:53,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-12-04T14:58:53,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 156 2024-12-04T14:58:53,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 153 2024-12-04T14:58:53,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 156 2024-12-04T14:58:53,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T14:58:53,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2169 2024-12-04T14:58:53,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2027 2024-12-04T14:58:53,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T14:58:53,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1005 2024-12-04T14:58:53,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3008 rows 2024-12-04T14:58:53,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1023 2024-12-04T14:58:53,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3069 rows 2024-12-04T14:58:53,266 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T14:58:53,266 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T14:58:53,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:53,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:53,267 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:53,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T14:58:53,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T14:58:53,281 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324333280"}]},"ts":"1733324333280"} 2024-12-04T14:58:53,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T14:58:53,282 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T14:58:53,282 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T14:58:53,284 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T14:58:53,287 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fc4055584695509eee4923dfb5273a9, UNASSIGN}] 2024-12-04T14:58:53,288 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fc4055584695509eee4923dfb5273a9, UNASSIGN 2024-12-04T14:58:53,289 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=7fc4055584695509eee4923dfb5273a9, regionState=CLOSING, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:53,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fc4055584695509eee4923dfb5273a9, UNASSIGN because future has completed 2024-12-04T14:58:53,292 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T14:58:53,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801}] 2024-12-04T14:58:53,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=706 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/07269b59590b4faabf6fd3c467f9681f 2024-12-04T14:58:53,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/931a9061689c4f499e54d86c060222a4 is 50, key is test_row_0/B:col10/1733324331077/Put/seqid=0 2024-12-04T14:58:53,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741934_1110 (size=12301) 2024-12-04T14:58:53,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T14:58:53,448 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(122): Close 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:53,448 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T14:58:53,448 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1722): Closing 7fc4055584695509eee4923dfb5273a9, disabling compactions & flushes 2024-12-04T14:58:53,449 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:53,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T14:58:53,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=706 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/931a9061689c4f499e54d86c060222a4 2024-12-04T14:58:53,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/d504d31405c94b67ba9d658a0b3d3367 is 50, key is test_row_0/C:col10/1733324331077/Put/seqid=0 2024-12-04T14:58:53,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741935_1111 (size=12301) 2024-12-04T14:58:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T14:58:54,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=706 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/d504d31405c94b67ba9d658a0b3d3367 2024-12-04T14:58:54,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/07269b59590b4faabf6fd3c467f9681f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/07269b59590b4faabf6fd3c467f9681f 2024-12-04T14:58:54,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/07269b59590b4faabf6fd3c467f9681f, entries=150, sequenceid=706, filesize=12.0 K 2024-12-04T14:58:54,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/931a9061689c4f499e54d86c060222a4 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/931a9061689c4f499e54d86c060222a4 2024-12-04T14:58:54,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/931a9061689c4f499e54d86c060222a4, entries=150, sequenceid=706, filesize=12.0 K 2024-12-04T14:58:54,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/d504d31405c94b67ba9d658a0b3d3367 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d504d31405c94b67ba9d658a0b3d3367 2024-12-04T14:58:54,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d504d31405c94b67ba9d658a0b3d3367, entries=150, sequenceid=706, filesize=12.0 K 2024-12-04T14:58:54,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=6.71 KB/6870 for 7fc4055584695509eee4923dfb5273a9 in 1285ms, sequenceid=706, compaction requested=false 2024-12-04T14:58:54,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fc4055584695509eee4923dfb5273a9: 2024-12-04T14:58:54,239 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:54,239 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:54,239 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. after waiting 0 ms 2024-12-04T14:58:54,239 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:54,239 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(2902): Flushing 7fc4055584695509eee4923dfb5273a9 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-12-04T14:58:54,240 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=A 2024-12-04T14:58:54,240 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:54,240 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=B 2024-12-04T14:58:54,240 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:54,240 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fc4055584695509eee4923dfb5273a9, store=C 2024-12-04T14:58:54,240 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:58:54,243 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/fb53c404ff8a4cdb9615239d4082fbc6 is 50, key is test_row_1/A:col10/1733324333261/Put/seqid=0 2024-12-04T14:58:54,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741936_1112 (size=7415) 2024-12-04T14:58:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T14:58:54,650 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=711 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/fb53c404ff8a4cdb9615239d4082fbc6 2024-12-04T14:58:54,664 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/0858f0599e1044358de1a56bd3cba3e8 is 50, key is test_row_1/B:col10/1733324333261/Put/seqid=0 2024-12-04T14:58:54,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741937_1113 (size=7415) 2024-12-04T14:58:55,069 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=711 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/0858f0599e1044358de1a56bd3cba3e8 2024-12-04T14:58:55,084 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/0f7923cab5a140ff9c2a4a1f9484a2b6 is 50, key is test_row_1/C:col10/1733324333261/Put/seqid=0 2024-12-04T14:58:55,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741938_1114 (size=7415) 2024-12-04T14:58:55,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T14:58:55,493 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=711 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/0f7923cab5a140ff9c2a4a1f9484a2b6 2024-12-04T14:58:55,505 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/A/fb53c404ff8a4cdb9615239d4082fbc6 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/fb53c404ff8a4cdb9615239d4082fbc6 2024-12-04T14:58:55,513 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/fb53c404ff8a4cdb9615239d4082fbc6, entries=50, sequenceid=711, filesize=7.2 K 2024-12-04T14:58:55,515 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/B/0858f0599e1044358de1a56bd3cba3e8 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/0858f0599e1044358de1a56bd3cba3e8 2024-12-04T14:58:55,519 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/0858f0599e1044358de1a56bd3cba3e8, entries=50, sequenceid=711, filesize=7.2 K 2024-12-04T14:58:55,520 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/.tmp/C/0f7923cab5a140ff9c2a4a1f9484a2b6 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/0f7923cab5a140ff9c2a4a1f9484a2b6 2024-12-04T14:58:55,524 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/0f7923cab5a140ff9c2a4a1f9484a2b6, entries=50, sequenceid=711, filesize=7.2 K 2024-12-04T14:58:55,525 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 7fc4055584695509eee4923dfb5273a9 in 1286ms, sequenceid=711, compaction requested=true 2024-12-04T14:58:55,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/14a96173a36348dd89478c112150b3a4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/35cce42dade84e019923fd284dc37edb, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e76854045f1245a0bc2531322e4edee0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/b35c0ad0440e47dfa60b517fce5988d8, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/70e9cf57722045eca89793b53cab3175, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e4cfb410f2a347428b72589277869c0f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/2a48cb6ad136411d879bde76f0f18354, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/62d0a4c31fd648ea90f86da925dde2ac, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/ee9097f1c06c4b7fac0916ae87186d4e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/57f212050fcd417fb1b2989b4fc38256, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/28a5a0df1c754150aaae6ef4df057fb0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/faa8d40417a042e981a48c1c148816e4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e38636b25416446b88587d9abafc8208, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/6974c2d66eba4971ad19819192771143, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/f9a024da50714d9489fde75c92668d27, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/17f2da4642724d9f98879cd72befe71d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/beaf155425a44e1e9a069a6cb024bdfe, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/2f736fcc69dd467fa960ccbc27beed89, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/7da7645bc33442d69768cb3595b842c9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/c69d1748bc6e42bcb7d8aa8935dea403, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/c263d186b4dc467db9da36660661d2cc, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/b538202e572e4ce78d006a655601cd85, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e6fb3f8cf5e84854bcf1d3500b9cc0dc, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/5042a8d5692043d38d9977ad6a4b36ae, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/6ea8b61e25b143279fb864f544e6efb9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/f7bbd19a907a41a59d512a334790127d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/2f52fe8b6cc8488a8e97ede8f19e200f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/981d5788d67b415a80d12585c8adce4b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/896d3e2351d54f838b4c49d2e41b5158, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/3ac0cf51043f494a9842b3254198c910, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/15bafea960994934a434b4bef35980d4] to archive 2024-12-04T14:58:55,529 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:58:55,534 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/14a96173a36348dd89478c112150b3a4 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/14a96173a36348dd89478c112150b3a4 2024-12-04T14:58:55,536 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/35cce42dade84e019923fd284dc37edb to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/35cce42dade84e019923fd284dc37edb 2024-12-04T14:58:55,538 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e76854045f1245a0bc2531322e4edee0 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e76854045f1245a0bc2531322e4edee0 2024-12-04T14:58:55,539 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/b35c0ad0440e47dfa60b517fce5988d8 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/b35c0ad0440e47dfa60b517fce5988d8 2024-12-04T14:58:55,540 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/70e9cf57722045eca89793b53cab3175 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/70e9cf57722045eca89793b53cab3175 2024-12-04T14:58:55,542 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e4cfb410f2a347428b72589277869c0f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e4cfb410f2a347428b72589277869c0f 2024-12-04T14:58:55,544 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/2a48cb6ad136411d879bde76f0f18354 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/2a48cb6ad136411d879bde76f0f18354 2024-12-04T14:58:55,545 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/62d0a4c31fd648ea90f86da925dde2ac to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/62d0a4c31fd648ea90f86da925dde2ac 2024-12-04T14:58:55,546 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/ee9097f1c06c4b7fac0916ae87186d4e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/ee9097f1c06c4b7fac0916ae87186d4e 2024-12-04T14:58:55,548 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/57f212050fcd417fb1b2989b4fc38256 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/57f212050fcd417fb1b2989b4fc38256 2024-12-04T14:58:55,549 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/28a5a0df1c754150aaae6ef4df057fb0 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/28a5a0df1c754150aaae6ef4df057fb0 2024-12-04T14:58:55,551 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/faa8d40417a042e981a48c1c148816e4 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/faa8d40417a042e981a48c1c148816e4 2024-12-04T14:58:55,552 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e38636b25416446b88587d9abafc8208 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e38636b25416446b88587d9abafc8208 2024-12-04T14:58:55,553 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/6974c2d66eba4971ad19819192771143 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/6974c2d66eba4971ad19819192771143 2024-12-04T14:58:55,555 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/f9a024da50714d9489fde75c92668d27 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/f9a024da50714d9489fde75c92668d27 2024-12-04T14:58:55,556 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/17f2da4642724d9f98879cd72befe71d to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/17f2da4642724d9f98879cd72befe71d 2024-12-04T14:58:55,557 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/beaf155425a44e1e9a069a6cb024bdfe to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/beaf155425a44e1e9a069a6cb024bdfe 2024-12-04T14:58:55,559 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/2f736fcc69dd467fa960ccbc27beed89 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/2f736fcc69dd467fa960ccbc27beed89 2024-12-04T14:58:55,560 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/7da7645bc33442d69768cb3595b842c9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/7da7645bc33442d69768cb3595b842c9 2024-12-04T14:58:55,561 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/c69d1748bc6e42bcb7d8aa8935dea403 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/c69d1748bc6e42bcb7d8aa8935dea403 2024-12-04T14:58:55,562 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/c263d186b4dc467db9da36660661d2cc to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/c263d186b4dc467db9da36660661d2cc 2024-12-04T14:58:55,563 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/b538202e572e4ce78d006a655601cd85 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/b538202e572e4ce78d006a655601cd85 2024-12-04T14:58:55,564 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e6fb3f8cf5e84854bcf1d3500b9cc0dc to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/e6fb3f8cf5e84854bcf1d3500b9cc0dc 2024-12-04T14:58:55,565 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/5042a8d5692043d38d9977ad6a4b36ae to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/5042a8d5692043d38d9977ad6a4b36ae 2024-12-04T14:58:55,566 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/6ea8b61e25b143279fb864f544e6efb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/6ea8b61e25b143279fb864f544e6efb9 2024-12-04T14:58:55,567 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/f7bbd19a907a41a59d512a334790127d to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/f7bbd19a907a41a59d512a334790127d 2024-12-04T14:58:55,568 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/2f52fe8b6cc8488a8e97ede8f19e200f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/2f52fe8b6cc8488a8e97ede8f19e200f 2024-12-04T14:58:55,569 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/981d5788d67b415a80d12585c8adce4b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/981d5788d67b415a80d12585c8adce4b 2024-12-04T14:58:55,570 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/896d3e2351d54f838b4c49d2e41b5158 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/896d3e2351d54f838b4c49d2e41b5158 2024-12-04T14:58:55,571 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/3ac0cf51043f494a9842b3254198c910 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/3ac0cf51043f494a9842b3254198c910 2024-12-04T14:58:55,572 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/15bafea960994934a434b4bef35980d4 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/15bafea960994934a434b4bef35980d4 2024-12-04T14:58:55,583 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/7a3881a81f83456ca46955f50f3a6312, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/3f843abf7a4246089691886922a9b5e4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/48ba29afd30b49ba9a5605ef9ebbc0cf, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/49e84e9317a9494083825557acbe6f0e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/a5c295b156764dc2b75a34692cfed6f1, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/db11f686763a4ffc93db4329e7975eeb, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/c7bbf96895ff4c728067b408813bcc1c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/af1c86342cb14b9d89e4d4c73972a97b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/81b28890700e4b96a5f03a5125de73a2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/92adb45f3c2b47d3bdd64b6df1923c5e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/96100a003a754b07af7cd00ea7cbfaa5, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/21407d91f2ce48d8a46f5ecaeca1b244, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/49dc5e975ce84f3985e2542bb0b4401a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/e6a1c61d61d842549c4e8dd10b27e952, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/cc625bbb7cd64d06b0a3a5ae23b6e493, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/ddfc9c8ed539438b81d00dcf995f8b6f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/6304a6836eaa4666a8853e086ddd5989, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/4d2eefc8585248fc8912cbae615acdf8, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/4c45b4656a8748f2a04e4ca812ad2812, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/45d092ad9757459bb5411a2c02511cb4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/3fff403b022c45289cd918297c54337e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/383d4c0a3c9a41b3ae0cc97bfd20deb2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/ae109fd14792451aacf5f99e7d458972, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/fc679eba960c45ff846cec0f187f9807, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/1c76d61609fa416ca47850d6e81553de, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/3a85cf6dd0424e4980e5c2553371725a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/c23e65755bb2430886a77aa9d4261490, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/e7d4d172081343e698127544e4cb4aa4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/404a9727bbd744fa9917e51f59851b3d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/29f4fcb505c243068ba3764fe9f589c9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/956bbf97a5f14243853de91234a07dd8] to archive 2024-12-04T14:58:55,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:58:55,586 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/7a3881a81f83456ca46955f50f3a6312 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/7a3881a81f83456ca46955f50f3a6312 2024-12-04T14:58:55,587 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/3f843abf7a4246089691886922a9b5e4 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/3f843abf7a4246089691886922a9b5e4 2024-12-04T14:58:55,588 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/48ba29afd30b49ba9a5605ef9ebbc0cf to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/48ba29afd30b49ba9a5605ef9ebbc0cf 2024-12-04T14:58:55,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/49e84e9317a9494083825557acbe6f0e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/49e84e9317a9494083825557acbe6f0e 2024-12-04T14:58:55,590 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/a5c295b156764dc2b75a34692cfed6f1 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/a5c295b156764dc2b75a34692cfed6f1 2024-12-04T14:58:55,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/db11f686763a4ffc93db4329e7975eeb to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/db11f686763a4ffc93db4329e7975eeb 2024-12-04T14:58:55,593 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/c7bbf96895ff4c728067b408813bcc1c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/c7bbf96895ff4c728067b408813bcc1c 2024-12-04T14:58:55,593 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/af1c86342cb14b9d89e4d4c73972a97b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/af1c86342cb14b9d89e4d4c73972a97b 2024-12-04T14:58:55,595 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/81b28890700e4b96a5f03a5125de73a2 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/81b28890700e4b96a5f03a5125de73a2 2024-12-04T14:58:55,596 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/92adb45f3c2b47d3bdd64b6df1923c5e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/92adb45f3c2b47d3bdd64b6df1923c5e 2024-12-04T14:58:55,597 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/96100a003a754b07af7cd00ea7cbfaa5 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/96100a003a754b07af7cd00ea7cbfaa5 2024-12-04T14:58:55,597 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/21407d91f2ce48d8a46f5ecaeca1b244 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/21407d91f2ce48d8a46f5ecaeca1b244 2024-12-04T14:58:55,598 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/49dc5e975ce84f3985e2542bb0b4401a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/49dc5e975ce84f3985e2542bb0b4401a 2024-12-04T14:58:55,599 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/e6a1c61d61d842549c4e8dd10b27e952 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/e6a1c61d61d842549c4e8dd10b27e952 2024-12-04T14:58:55,600 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/cc625bbb7cd64d06b0a3a5ae23b6e493 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/cc625bbb7cd64d06b0a3a5ae23b6e493 2024-12-04T14:58:55,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/ddfc9c8ed539438b81d00dcf995f8b6f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/ddfc9c8ed539438b81d00dcf995f8b6f 2024-12-04T14:58:55,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/6304a6836eaa4666a8853e086ddd5989 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/6304a6836eaa4666a8853e086ddd5989 2024-12-04T14:58:55,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/4d2eefc8585248fc8912cbae615acdf8 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/4d2eefc8585248fc8912cbae615acdf8 2024-12-04T14:58:55,603 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/4c45b4656a8748f2a04e4ca812ad2812 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/4c45b4656a8748f2a04e4ca812ad2812 2024-12-04T14:58:55,604 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/45d092ad9757459bb5411a2c02511cb4 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/45d092ad9757459bb5411a2c02511cb4 2024-12-04T14:58:55,605 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/3fff403b022c45289cd918297c54337e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/3fff403b022c45289cd918297c54337e 2024-12-04T14:58:55,606 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/383d4c0a3c9a41b3ae0cc97bfd20deb2 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/383d4c0a3c9a41b3ae0cc97bfd20deb2 2024-12-04T14:58:55,607 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/ae109fd14792451aacf5f99e7d458972 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/ae109fd14792451aacf5f99e7d458972 2024-12-04T14:58:55,607 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/fc679eba960c45ff846cec0f187f9807 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/fc679eba960c45ff846cec0f187f9807 2024-12-04T14:58:55,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/1c76d61609fa416ca47850d6e81553de to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/1c76d61609fa416ca47850d6e81553de 2024-12-04T14:58:55,609 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/3a85cf6dd0424e4980e5c2553371725a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/3a85cf6dd0424e4980e5c2553371725a 2024-12-04T14:58:55,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/c23e65755bb2430886a77aa9d4261490 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/c23e65755bb2430886a77aa9d4261490 2024-12-04T14:58:55,611 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/e7d4d172081343e698127544e4cb4aa4 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/e7d4d172081343e698127544e4cb4aa4 2024-12-04T14:58:55,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/404a9727bbd744fa9917e51f59851b3d to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/404a9727bbd744fa9917e51f59851b3d 2024-12-04T14:58:55,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/29f4fcb505c243068ba3764fe9f589c9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/29f4fcb505c243068ba3764fe9f589c9 2024-12-04T14:58:55,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/956bbf97a5f14243853de91234a07dd8 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/956bbf97a5f14243853de91234a07dd8 2024-12-04T14:58:55,614 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/607fd62f9ad04a08bcb0f4dfaf384af9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/5a18909aa0794c4180e06dede9ada2ad, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/de90ccc7605e41eeae431b85bd479d1e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/31afd7f434574f1aac5de6af5b7c1016, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/74c6b81fc322453890bab73ca1c86048, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/36cea9cefb7044a1a68faa819a0d9149, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/8d5968f923d346a38222de455c4dd448, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/f3fe9f09221e41a58027a23ca71e9211, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/53c6c9aa31864503a812e17da54ed89f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d819d1f2d32c40d1ab5fa8c34b79f855, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/0acb978d2e7541028508ffaaadc560d2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/ae300b003fc04870a54b2f13e82bb25e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/fe9ccc378f46403d9a9e093bec9e8f5f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/0f75f6178a494f67acdf5d85999e411a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/3641d1c42bfe49c59357130b984e7b99, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/a39f37b74372455fbd9f301026abffa7, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/eded0b01665247a68d81dfd37d7648eb, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/36bd7cd66f4849038ea632780abf486b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/ca7b84ab1dcf4643945f0d0f6df8b40d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/f1f8a691a77f4fb4b38c410f6ebba1ed, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/7c0b4da0d4834acdb10785ba0a2c0c2c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/191d57e0594740f9b92e06963baf2f9f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/1775f8e86ba84e4baef4ae570f5e2af7, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/6a422d6d86554bf8b9c902b85eee8d55, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/0d7bdf3ec10948a6a674e85f38cf1422, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/1bd2fc57d97c46c6adbe641bf4c7537c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/cb845609f9ba4ac68d8b06c4293e8470, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/48c44b837fe6436d9e586979f994f9af, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d7ed5645606d4b1ebfa6d51a17b4dcc2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d7eaaa75326e4699a0bc4d0f7f987d6d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d2a2013868fa4aacb09e00b349a2e304] to archive 2024-12-04T14:58:55,615 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:58:55,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/607fd62f9ad04a08bcb0f4dfaf384af9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/607fd62f9ad04a08bcb0f4dfaf384af9 2024-12-04T14:58:55,617 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/5a18909aa0794c4180e06dede9ada2ad to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/5a18909aa0794c4180e06dede9ada2ad 2024-12-04T14:58:55,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/de90ccc7605e41eeae431b85bd479d1e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/de90ccc7605e41eeae431b85bd479d1e 2024-12-04T14:58:55,619 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/31afd7f434574f1aac5de6af5b7c1016 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/31afd7f434574f1aac5de6af5b7c1016 2024-12-04T14:58:55,620 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/74c6b81fc322453890bab73ca1c86048 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/74c6b81fc322453890bab73ca1c86048 2024-12-04T14:58:55,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/36cea9cefb7044a1a68faa819a0d9149 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/36cea9cefb7044a1a68faa819a0d9149 2024-12-04T14:58:55,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/8d5968f923d346a38222de455c4dd448 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/8d5968f923d346a38222de455c4dd448 2024-12-04T14:58:55,623 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/f3fe9f09221e41a58027a23ca71e9211 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/f3fe9f09221e41a58027a23ca71e9211 2024-12-04T14:58:55,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/53c6c9aa31864503a812e17da54ed89f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/53c6c9aa31864503a812e17da54ed89f 2024-12-04T14:58:55,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d819d1f2d32c40d1ab5fa8c34b79f855 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d819d1f2d32c40d1ab5fa8c34b79f855 2024-12-04T14:58:55,625 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/0acb978d2e7541028508ffaaadc560d2 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/0acb978d2e7541028508ffaaadc560d2 2024-12-04T14:58:55,626 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/ae300b003fc04870a54b2f13e82bb25e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/ae300b003fc04870a54b2f13e82bb25e 2024-12-04T14:58:55,627 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/fe9ccc378f46403d9a9e093bec9e8f5f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/fe9ccc378f46403d9a9e093bec9e8f5f 2024-12-04T14:58:55,627 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/0f75f6178a494f67acdf5d85999e411a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/0f75f6178a494f67acdf5d85999e411a 2024-12-04T14:58:55,628 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/3641d1c42bfe49c59357130b984e7b99 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/3641d1c42bfe49c59357130b984e7b99 2024-12-04T14:58:55,629 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/a39f37b74372455fbd9f301026abffa7 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/a39f37b74372455fbd9f301026abffa7 2024-12-04T14:58:55,630 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/eded0b01665247a68d81dfd37d7648eb to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/eded0b01665247a68d81dfd37d7648eb 2024-12-04T14:58:55,631 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/36bd7cd66f4849038ea632780abf486b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/36bd7cd66f4849038ea632780abf486b 2024-12-04T14:58:55,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/ca7b84ab1dcf4643945f0d0f6df8b40d to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/ca7b84ab1dcf4643945f0d0f6df8b40d 2024-12-04T14:58:55,633 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/f1f8a691a77f4fb4b38c410f6ebba1ed to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/f1f8a691a77f4fb4b38c410f6ebba1ed 2024-12-04T14:58:55,633 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/7c0b4da0d4834acdb10785ba0a2c0c2c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/7c0b4da0d4834acdb10785ba0a2c0c2c 2024-12-04T14:58:55,634 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/191d57e0594740f9b92e06963baf2f9f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/191d57e0594740f9b92e06963baf2f9f 2024-12-04T14:58:55,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/1775f8e86ba84e4baef4ae570f5e2af7 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/1775f8e86ba84e4baef4ae570f5e2af7 2024-12-04T14:58:55,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/6a422d6d86554bf8b9c902b85eee8d55 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/6a422d6d86554bf8b9c902b85eee8d55 2024-12-04T14:58:55,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/0d7bdf3ec10948a6a674e85f38cf1422 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/0d7bdf3ec10948a6a674e85f38cf1422 2024-12-04T14:58:55,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/1bd2fc57d97c46c6adbe641bf4c7537c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/1bd2fc57d97c46c6adbe641bf4c7537c 2024-12-04T14:58:55,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/cb845609f9ba4ac68d8b06c4293e8470 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/cb845609f9ba4ac68d8b06c4293e8470 2024-12-04T14:58:55,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/48c44b837fe6436d9e586979f994f9af to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/48c44b837fe6436d9e586979f994f9af 2024-12-04T14:58:55,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d7ed5645606d4b1ebfa6d51a17b4dcc2 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d7ed5645606d4b1ebfa6d51a17b4dcc2 2024-12-04T14:58:55,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d7eaaa75326e4699a0bc4d0f7f987d6d to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d7eaaa75326e4699a0bc4d0f7f987d6d 2024-12-04T14:58:55,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d2a2013868fa4aacb09e00b349a2e304 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d2a2013868fa4aacb09e00b349a2e304 2024-12-04T14:58:55,648 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/recovered.edits/714.seqid, newMaxSeqId=714, maxSeqId=1 2024-12-04T14:58:55,649 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. 2024-12-04T14:58:55,649 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1676): Region close journal for 7fc4055584695509eee4923dfb5273a9: Waiting for close lock at 1733324333448Running coprocessor pre-close hooks at 1733324333448Disabling compacts and flushes for region at 1733324333448Disabling writes for close at 1733324334239 (+791 ms)Obtaining lock to block concurrent updates at 1733324334239Preparing flush snapshotting stores in 7fc4055584695509eee4923dfb5273a9 at 1733324334239Finished memstore snapshotting TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1733324334240 (+1 ms)Flushing stores of TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9. at 1733324334240Flushing 7fc4055584695509eee4923dfb5273a9/A: creating writer at 1733324334240Flushing 7fc4055584695509eee4923dfb5273a9/A: appending metadata at 1733324334243 (+3 ms)Flushing 7fc4055584695509eee4923dfb5273a9/A: closing flushed file at 1733324334243Flushing 7fc4055584695509eee4923dfb5273a9/B: creating writer at 1733324334661 (+418 ms)Flushing 7fc4055584695509eee4923dfb5273a9/B: appending metadata at 1733324334664 (+3 ms)Flushing 7fc4055584695509eee4923dfb5273a9/B: closing flushed file at 1733324334664Flushing 7fc4055584695509eee4923dfb5273a9/C: creating writer at 1733324335078 (+414 ms)Flushing 7fc4055584695509eee4923dfb5273a9/C: appending metadata at 1733324335083 (+5 ms)Flushing 7fc4055584695509eee4923dfb5273a9/C: closing flushed file at 1733324335083Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e84d69f: reopening flushed file at 1733324335503 (+420 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dc98f3a: reopening flushed file at 1733324335513 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@192f44ad: reopening flushed file at 1733324335519 (+6 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 7fc4055584695509eee4923dfb5273a9 in 1286ms, sequenceid=711, compaction requested=true at 1733324335525 (+6 ms)Writing region close event to WAL at 1733324335645 (+120 ms)Running coprocessor post-close hooks at 1733324335649 (+4 ms)Closed at 1733324335649 2024-12-04T14:58:55,651 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(157): Closed 7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:55,652 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=7fc4055584695509eee4923dfb5273a9, regionState=CLOSED 2024-12-04T14:58:55,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 because future has completed 2024-12-04T14:58:55,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-04T14:58:55,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseRegionProcedure 7fc4055584695509eee4923dfb5273a9, server=d214edae7bd2,36649,1733324304801 in 2.3630 sec 2024-12-04T14:58:55,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-04T14:58:55,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fc4055584695509eee4923dfb5273a9, UNASSIGN in 2.3700 sec 2024-12-04T14:58:55,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-04T14:58:55,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3770 sec 2024-12-04T14:58:55,663 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324335663"}]},"ts":"1733324335663"} 2024-12-04T14:58:55,665 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T14:58:55,665 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T14:58:55,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3890 sec 2024-12-04T14:58:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T14:58:57,440 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:57,443 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T14:58:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:58:57,447 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:58:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T14:58:57,448 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=25, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:58:57,453 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:57,457 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A, FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B, FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C, FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/recovered.edits] 2024-12-04T14:58:57,461 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/07269b59590b4faabf6fd3c467f9681f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/07269b59590b4faabf6fd3c467f9681f 2024-12-04T14:58:57,463 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/989fe5384b0b4ffeb00f39d63a395e24 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/989fe5384b0b4ffeb00f39d63a395e24 2024-12-04T14:58:57,464 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/fb53c404ff8a4cdb9615239d4082fbc6 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/A/fb53c404ff8a4cdb9615239d4082fbc6 2024-12-04T14:58:57,468 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/0858f0599e1044358de1a56bd3cba3e8 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/0858f0599e1044358de1a56bd3cba3e8 2024-12-04T14:58:57,470 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/6658ba3b708d45c08a9ab50eb1c717c3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/6658ba3b708d45c08a9ab50eb1c717c3 2024-12-04T14:58:57,471 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/931a9061689c4f499e54d86c060222a4 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/B/931a9061689c4f499e54d86c060222a4 2024-12-04T14:58:57,475 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/0f7923cab5a140ff9c2a4a1f9484a2b6 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/0f7923cab5a140ff9c2a4a1f9484a2b6 2024-12-04T14:58:57,476 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/c467797204c94b62aa9ee10ad569d4a9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/c467797204c94b62aa9ee10ad569d4a9 2024-12-04T14:58:57,478 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d504d31405c94b67ba9d658a0b3d3367 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/C/d504d31405c94b67ba9d658a0b3d3367 2024-12-04T14:58:57,481 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/recovered.edits/714.seqid to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9/recovered.edits/714.seqid 2024-12-04T14:58:57,481 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/7fc4055584695509eee4923dfb5273a9 2024-12-04T14:58:57,481 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T14:58:57,486 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=25, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:58:57,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-04T14:58:57,495 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T14:58:57,520 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T14:58:57,521 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=25, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:58:57,521 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T14:58:57,521 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733324337521"}]},"ts":"9223372036854775807"} 2024-12-04T14:58:57,525 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T14:58:57,525 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7fc4055584695509eee4923dfb5273a9, NAME => 'TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T14:58:57,525 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T14:58:57,525 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733324337525"}]},"ts":"9223372036854775807"} 2024-12-04T14:58:57,528 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T14:58:57,529 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=25, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:58:57,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 86 msec 2024-12-04T14:58:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T14:58:57,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T14:58:57,551 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:57,561 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;d214edae7bd2:36649-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-325510507_22 at /127.0.0.1:52100 [Waiting for operation #68] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1045188705_22 at /127.0.0.1:45540 [Waiting for operation #293] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1045188705_22 at /127.0.0.1:54338 [Waiting for operation #271] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=661 (was 473) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8921 (was 9480) 2024-12-04T14:58:57,571 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=661, ProcessCount=11, AvailableMemoryMB=8920 2024-12-04T14:58:57,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T14:58:57,573 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:58:57,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T14:58:57,575 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T14:58:57,575 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:57,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 26 2024-12-04T14:58:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-04T14:58:57,576 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T14:58:57,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741939_1115 (size=1061) 2024-12-04T14:58:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-04T14:58:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-04T14:58:57,990 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee 2024-12-04T14:58:57,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741940_1116 (size=53) 2024-12-04T14:58:58,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-04T14:58:58,401 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:58:58,401 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 47aad6a6d662b77a47b7fe8495dd267c, disabling compactions & flushes 2024-12-04T14:58:58,401 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:58:58,401 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:58:58,401 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. after waiting 0 ms 2024-12-04T14:58:58,401 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:58:58,402 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:58:58,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 47aad6a6d662b77a47b7fe8495dd267c: Waiting for close lock at 1733324338401Disabling compacts and flushes for region at 1733324338401Disabling writes for close at 1733324338401Writing region close event to WAL at 1733324338401Closed at 1733324338401 2024-12-04T14:58:58,405 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T14:58:58,406 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733324338405"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733324338405"}]},"ts":"1733324338405"} 2024-12-04T14:58:58,409 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T14:58:58,410 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T14:58:58,410 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324338410"}]},"ts":"1733324338410"} 2024-12-04T14:58:58,412 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T14:58:58,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47aad6a6d662b77a47b7fe8495dd267c, ASSIGN}] 2024-12-04T14:58:58,414 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47aad6a6d662b77a47b7fe8495dd267c, ASSIGN 2024-12-04T14:58:58,415 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47aad6a6d662b77a47b7fe8495dd267c, ASSIGN; state=OFFLINE, location=d214edae7bd2,36649,1733324304801; forceNewPlan=false, retain=false 2024-12-04T14:58:58,566 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=47aad6a6d662b77a47b7fe8495dd267c, regionState=OPENING, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:58,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47aad6a6d662b77a47b7fe8495dd267c, ASSIGN because future has completed 2024-12-04T14:58:58,573 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801}] 2024-12-04T14:58:58,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-04T14:58:58,762 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:58:58,762 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:58:58,762 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:58:58,762 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:58:58,763 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:58:58,763 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:58:58,764 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:58:58,766 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T14:58:58,767 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47aad6a6d662b77a47b7fe8495dd267c columnFamilyName A 2024-12-04T14:58:58,767 DEBUG [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:58,768 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] regionserver.HStore(327): Store=47aad6a6d662b77a47b7fe8495dd267c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:58:58,768 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:58:58,770 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T14:58:58,770 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47aad6a6d662b77a47b7fe8495dd267c columnFamilyName B 2024-12-04T14:58:58,770 DEBUG [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:58,771 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] regionserver.HStore(327): Store=47aad6a6d662b77a47b7fe8495dd267c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:58:58,771 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:58:58,772 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T14:58:58,772 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47aad6a6d662b77a47b7fe8495dd267c columnFamilyName C 2024-12-04T14:58:58,772 DEBUG [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:58,773 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] regionserver.HStore(327): Store=47aad6a6d662b77a47b7fe8495dd267c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:58:58,773 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:58:58,773 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:58:58,774 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:58:58,774 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:58:58,775 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:58:58,775 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:58:58,776 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:58:58,777 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:58:58,779 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:58:58,779 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 47aad6a6d662b77a47b7fe8495dd267c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69102592, jitterRate=0.0297088623046875}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:58:58,779 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:58:58,780 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 47aad6a6d662b77a47b7fe8495dd267c: Running coprocessor pre-open hook at 1733324338763Writing region info on filesystem at 1733324338763Initializing all the Stores at 1733324338764 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324338764Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324338764Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324338764Cleaning up temporary data from old regions at 1733324338775 (+11 ms)Running coprocessor post-open hooks at 1733324338779 (+4 ms)Region opened successfully at 1733324338780 (+1 ms) 2024-12-04T14:58:58,781 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., pid=28, masterSystemTime=1733324338753 2024-12-04T14:58:58,782 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:58:58,782 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:58:58,783 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=47aad6a6d662b77a47b7fe8495dd267c, regionState=OPEN, openSeqNum=2, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T14:58:58,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 because future has completed 2024-12-04T14:58:58,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-04T14:58:58,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 in 214 msec 2024-12-04T14:58:58,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-04T14:58:58,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47aad6a6d662b77a47b7fe8495dd267c, ASSIGN in 377 msec 2024-12-04T14:58:58,792 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T14:58:58,792 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324338792"}]},"ts":"1733324338792"} 2024-12-04T14:58:58,793 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T14:58:58,794 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T14:58:58,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2210 sec 2024-12-04T14:58:59,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-04T14:58:59,721 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:59,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e732f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:59,724 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:58:59,724 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:59,727 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:58:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:59,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:58:59,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f3e4ba4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:59,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:58:59,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:59,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:59,732 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:59,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d5b06de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:59,733 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:59,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:58:59,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:59,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:59,739 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T14:58:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is d214edae7bd2,39449,1733324304115 2024-12-04T14:58:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1b7a241d 2024-12-04T14:58:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T14:58:59,740 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T14:58:59,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T14:58:59,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:58:59,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-04T14:58:59,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741941_1117 (size=1097) 2024-12-04T14:59:00,169 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-04T14:59:00,169 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-04T14:59:00,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T14:59:00,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47aad6a6d662b77a47b7fe8495dd267c, REOPEN/MOVE}] 2024-12-04T14:59:00,185 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47aad6a6d662b77a47b7fe8495dd267c, REOPEN/MOVE 2024-12-04T14:59:00,186 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=47aad6a6d662b77a47b7fe8495dd267c, regionState=CLOSING, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:00,187 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39449 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=d214edae7bd2,36649,1733324304801, table=TestAcidGuarantees, region=47aad6a6d662b77a47b7fe8495dd267c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-04T14:59:00,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47aad6a6d662b77a47b7fe8495dd267c, REOPEN/MOVE because future has completed 2024-12-04T14:59:00,189 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T14:59:00,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801}] 2024-12-04T14:59:00,342 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:00,343 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T14:59:00,343 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 47aad6a6d662b77a47b7fe8495dd267c, disabling compactions & flushes 2024-12-04T14:59:00,343 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:00,343 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:00,343 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. after waiting 0 ms 2024-12-04T14:59:00,343 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:00,352 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-04T14:59:00,352 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:00,352 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 47aad6a6d662b77a47b7fe8495dd267c: Waiting for close lock at 1733324340343Running coprocessor pre-close hooks at 1733324340343Disabling compacts and flushes for region at 1733324340343Disabling writes for close at 1733324340343Writing region close event to WAL at 1733324340344 (+1 ms)Running coprocessor post-close hooks at 1733324340352 (+8 ms)Closed at 1733324340352 2024-12-04T14:59:00,353 WARN [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionServer(3299): Not adding moved region record: 47aad6a6d662b77a47b7fe8495dd267c to self. 2024-12-04T14:59:00,355 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:00,355 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=47aad6a6d662b77a47b7fe8495dd267c, regionState=CLOSED 2024-12-04T14:59:00,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 because future has completed 2024-12-04T14:59:00,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-04T14:59:00,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 in 169 msec 2024-12-04T14:59:00,360 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47aad6a6d662b77a47b7fe8495dd267c, REOPEN/MOVE; state=CLOSED, location=d214edae7bd2,36649,1733324304801; forceNewPlan=false, retain=true 2024-12-04T14:59:00,511 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=47aad6a6d662b77a47b7fe8495dd267c, regionState=OPENING, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:00,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47aad6a6d662b77a47b7fe8495dd267c, REOPEN/MOVE because future has completed 2024-12-04T14:59:00,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801}] 2024-12-04T14:59:00,672 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:00,672 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:59:00,673 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:00,673 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:59:00,673 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:00,673 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:00,675 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:00,676 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T14:59:00,679 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47aad6a6d662b77a47b7fe8495dd267c columnFamilyName A 2024-12-04T14:59:00,681 DEBUG [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:00,682 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] regionserver.HStore(327): Store=47aad6a6d662b77a47b7fe8495dd267c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:59:00,682 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:00,683 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T14:59:00,683 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47aad6a6d662b77a47b7fe8495dd267c columnFamilyName B 2024-12-04T14:59:00,683 DEBUG [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:00,683 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] regionserver.HStore(327): Store=47aad6a6d662b77a47b7fe8495dd267c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:59:00,683 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:00,684 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T14:59:00,684 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47aad6a6d662b77a47b7fe8495dd267c columnFamilyName C 2024-12-04T14:59:00,684 DEBUG [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:00,684 INFO [StoreOpener-47aad6a6d662b77a47b7fe8495dd267c-1 {}] regionserver.HStore(327): Store=47aad6a6d662b77a47b7fe8495dd267c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:59:00,685 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:00,685 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:00,685 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:00,687 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:00,688 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:00,688 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:00,688 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:59:00,689 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:00,690 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 47aad6a6d662b77a47b7fe8495dd267c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74718739, jitterRate=0.11339597404003143}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:59:00,690 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:00,690 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 47aad6a6d662b77a47b7fe8495dd267c: Running coprocessor pre-open hook at 1733324340673Writing region info on filesystem at 1733324340673Initializing all the Stores at 1733324340674 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324340674Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324340674Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324340674Cleaning up temporary data from old regions at 1733324340688 (+14 ms)Running coprocessor post-open hooks at 1733324340690 (+2 ms)Region opened successfully at 1733324340690 2024-12-04T14:59:00,691 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., pid=33, masterSystemTime=1733324340669 2024-12-04T14:59:00,693 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:00,693 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:00,693 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=47aad6a6d662b77a47b7fe8495dd267c, regionState=OPEN, openSeqNum=5, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:00,695 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 because future has completed 2024-12-04T14:59:00,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=31 2024-12-04T14:59:00,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 in 180 msec 2024-12-04T14:59:00,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-04T14:59:00,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47aad6a6d662b77a47b7fe8495dd267c, REOPEN/MOVE in 513 msec 2024-12-04T14:59:00,702 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-04T14:59:00,702 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 524 msec 2024-12-04T14:59:00,706 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 957 msec 2024-12-04T14:59:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T14:59:00,708 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:00,709 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4716031f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,709 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:59:00,709 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:59:00,710 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:59:00,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:59:00,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:59:00,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bfb722b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:59:00,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:59:00,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:00,712 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:59:00,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53c7da6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d3ba43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,713 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:59:00,713 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:59:00,714 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:59:00,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:59:00,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:59:00,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c31ef4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:59:00,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:59:00,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:00,716 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:59:00,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a14328a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71ccd683, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:59:00,717 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:59:00,718 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:59:00,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:59:00,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:59:00,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23591909, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:59:00,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:59:00,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:00,720 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:59:00,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@695bede0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,721 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47250f91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,721 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:59:00,721 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:59:00,722 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:59:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:59:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:59:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f3969d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:59:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:59:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:00,724 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:59:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f0d576d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@598c3817, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:59:00,726 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:59:00,726 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:59:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:59:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:59:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2ab1d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:59:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:59:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:00,728 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:59:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66df0a84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,729 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed913a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,729 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:59:00,730 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:59:00,730 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:59:00,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:59:00,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:59:00,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@333186ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:59:00,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:59:00,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:00,732 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:59:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51e90f81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49519de2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,733 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:59:00,734 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:59:00,734 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:59:00,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:59:00,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:59:00,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d9c7e1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:59:00,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:59:00,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:00,736 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:59:00,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a0d289b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16801009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,737 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:59:00,737 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:59:00,738 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:59:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:59:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:59:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bda395f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:59:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:59:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:00,739 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52878, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:59:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71aa1874, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,740 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7183633a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,740 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:59:00,741 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:59:00,741 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:59:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:59:00,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:59:00,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da5c916, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:59:00,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:59:00,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:00,743 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52886, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:59:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7864e981, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:00,744 DEBUG [Thread-602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:59:00,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:59:00,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:59:00,746 DEBUG [Thread-604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:59:00,746 DEBUG [Thread-608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:59:00,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:59:00,747 DEBUG [Thread-611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:59:00,747 DEBUG [Thread-615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:59:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:59:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:59:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:59:00,748 DEBUG [Thread-606 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:59:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:59:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:59:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:59:00,749 DEBUG [Thread-617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:59:00,749 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:59:00,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:59:00,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:59:00,750 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:59:00,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:59:00,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:59:00,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:59:00,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:59:00,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T14:59:00,752 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:59:00,752 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:59:00,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T14:59:00,753 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:59:00,753 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:59:00,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T14:59:00,755 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:59:00,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T14:59:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-12-04T14:59:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T14:59:00,757 DEBUG [Thread-613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:59:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T14:59:00,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:59:00,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T14:59:00,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T14:59:00,759 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:59:00,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:59:00,761 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:59:00,762 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:59:00,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:59:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T14:59:00,771 DEBUG [Thread-600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:59:00,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:59:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:59:00,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:59:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T14:59:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:00,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47aad6a6d662b77a47b7fe8495dd267c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T14:59:00,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=A 2024-12-04T14:59:00,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:00,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=B 2024-12-04T14:59:00,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:00,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=C 2024-12-04T14:59:00,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:00,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39330 deadline: 1733324400821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:00,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733324340805, max timestamp=1733324340826Num uniques 859; 2024-12-04T14:59:00,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-04T14:59:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:00,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733324340805, max timestamp=1733324340826Num uniques 859; 2024-12-04T14:59:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:00,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-04T14:59:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:00,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:00,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39338 deadline: 1733324400823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:00,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:00,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:00,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:00,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733324340805, max timestamp=1733324340826Num uniques 859; 2024-12-04T14:59:00,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-04T14:59:00,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120455b8c95f4e424963bc7e44c8f90e8e16_47aad6a6d662b77a47b7fe8495dd267c is 50, key is test_row_0/A:col10/1733324340775/Put/seqid=0 2024-12-04T14:59:00,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741942_1118 (size=14594) 2024-12-04T14:59:00,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T14:59:00,860 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:00,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:00,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324400862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:00,869 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120455b8c95f4e424963bc7e44c8f90e8e16_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120455b8c95f4e424963bc7e44c8f90e8e16_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:00,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:00,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:00,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:00,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:00,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324400867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:00,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:00,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:00,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:00,873 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/776ddd523f0748da9eb69c5cd6e202f5, store: [table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:00,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733324400869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:00,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/776ddd523f0748da9eb69c5cd6e202f5 is 175, key is test_row_0/A:col10/1733324340775/Put/seqid=0 2024-12-04T14:59:00,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741943_1119 (size=39549) 2024-12-04T14:59:00,906 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/776ddd523f0748da9eb69c5cd6e202f5 2024-12-04T14:59:00,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-04T14:59:00,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:00,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:00,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:00,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:00,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:00,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:00,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39330 deadline: 1733324400953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:00,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39338 deadline: 1733324400954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:00,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:00,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:00,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:00,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/3a505fc8e0834ea4a29b9d555b9f8c10 is 50, key is test_row_0/B:col10/1733324340775/Put/seqid=0 2024-12-04T14:59:00,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741944_1120 (size=12001) 2024-12-04T14:59:00,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/3a505fc8e0834ea4a29b9d555b9f8c10 2024-12-04T14:59:00,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:00,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324400980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:00,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:00,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:00,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:00,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733324400980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:00,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:00,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:00,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324400981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:00,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:00,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:01,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/c01e14c081c04805bf4ab65718927b20 is 50, key is test_row_0/C:col10/1733324340775/Put/seqid=0 2024-12-04T14:59:01,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741945_1121 (size=12001) 2024-12-04T14:59:01,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/c01e14c081c04805bf4ab65718927b20 2024-12-04T14:59:01,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/776ddd523f0748da9eb69c5cd6e202f5 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/776ddd523f0748da9eb69c5cd6e202f5 2024-12-04T14:59:01,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/776ddd523f0748da9eb69c5cd6e202f5, entries=200, sequenceid=16, filesize=38.6 K 2024-12-04T14:59:01,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/3a505fc8e0834ea4a29b9d555b9f8c10 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/3a505fc8e0834ea4a29b9d555b9f8c10 2024-12-04T14:59:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T14:59:01,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/3a505fc8e0834ea4a29b9d555b9f8c10, entries=150, sequenceid=16, filesize=11.7 K 2024-12-04T14:59:01,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/c01e14c081c04805bf4ab65718927b20 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/c01e14c081c04805bf4ab65718927b20 2024-12-04T14:59:01,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-04T14:59:01,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:01,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:01,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:01,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:01,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:01,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/c01e14c081c04805bf4ab65718927b20, entries=150, sequenceid=16, filesize=11.7 K 2024-12-04T14:59:01,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 47aad6a6d662b77a47b7fe8495dd267c in 312ms, sequenceid=16, compaction requested=false 2024-12-04T14:59:01,090 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-04T14:59:01,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:01,169 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T14:59:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:01,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47aad6a6d662b77a47b7fe8495dd267c 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-12-04T14:59:01,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=A 2024-12-04T14:59:01,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:01,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=B 2024-12-04T14:59:01,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:01,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=C 2024-12-04T14:59:01,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:01,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412044ac74d3932b84bae92494c66139899c3_47aad6a6d662b77a47b7fe8495dd267c is 50, key is test_row_0/A:col10/1733324341165/Put/seqid=0 2024-12-04T14:59:01,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741946_1122 (size=14594) 2024-12-04T14:59:01,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324401201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:01,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:01,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39338 deadline: 1733324401204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:01,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:01,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324401209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:01,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:01,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39330 deadline: 1733324401209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:01,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:01,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733324401215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:01,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:01,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-04T14:59:01,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:01,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:01,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:01,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:01,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:01,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:01,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324401330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:01,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:01,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39338 deadline: 1733324401330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:01,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:01,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:01,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39330 deadline: 1733324401330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:01,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:01,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:01,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:01,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733324401331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:01,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T14:59:01,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-04T14:59:01,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:01,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:01,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:01,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:01,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:01,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324401520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:01,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:01,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324401540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:01,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733324401541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:01,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:01,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:01,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39338 deadline: 1733324401543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:01,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39330 deadline: 1733324401543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:01,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:01,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-04T14:59:01,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:01,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:01,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:01,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:01,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:01,609 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:01,616 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412044ac74d3932b84bae92494c66139899c3_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412044ac74d3932b84bae92494c66139899c3_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:01,617 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/050fe7463e73437894b5a69e45082d7f, store: [table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:01,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/050fe7463e73437894b5a69e45082d7f is 175, key is test_row_0/A:col10/1733324341165/Put/seqid=0 2024-12-04T14:59:01,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741947_1123 (size=39549) 2024-12-04T14:59:01,626 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/050fe7463e73437894b5a69e45082d7f 2024-12-04T14:59:01,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/526d2ad5271743e58060da0d08d83c9a is 50, key is test_row_0/B:col10/1733324341165/Put/seqid=0 2024-12-04T14:59:01,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741948_1124 (size=12001) 2024-12-04T14:59:01,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/526d2ad5271743e58060da0d08d83c9a 2024-12-04T14:59:01,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/13fbad04d2ef4b2cbf9d040695385791 is 50, key is test_row_0/C:col10/1733324341165/Put/seqid=0 2024-12-04T14:59:01,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-04T14:59:01,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:01,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:01,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:01,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:01,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:01,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741949_1125 (size=12001) 2024-12-04T14:59:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:01,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733324401851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:01,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39338 deadline: 1733324401851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:01,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:01,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39330 deadline: 1733324401851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:01,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324401851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:01,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-04T14:59:01,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:01,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:01,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:01,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:01,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T14:59:02,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-04T14:59:02,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:02,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:02,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:02,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:02,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:02,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324402030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:02,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:02,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/13fbad04d2ef4b2cbf9d040695385791 2024-12-04T14:59:02,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/050fe7463e73437894b5a69e45082d7f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/050fe7463e73437894b5a69e45082d7f 2024-12-04T14:59:02,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/050fe7463e73437894b5a69e45082d7f, entries=200, sequenceid=49, filesize=38.6 K 2024-12-04T14:59:02,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/526d2ad5271743e58060da0d08d83c9a as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/526d2ad5271743e58060da0d08d83c9a 2024-12-04T14:59:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/526d2ad5271743e58060da0d08d83c9a, entries=150, sequenceid=49, filesize=11.7 K 2024-12-04T14:59:02,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/13fbad04d2ef4b2cbf9d040695385791 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/13fbad04d2ef4b2cbf9d040695385791 2024-12-04T14:59:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/13fbad04d2ef4b2cbf9d040695385791, entries=150, sequenceid=49, filesize=11.7 K 2024-12-04T14:59:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=53.67 KB/54960 for 47aad6a6d662b77a47b7fe8495dd267c in 973ms, sequenceid=49, compaction requested=false 2024-12-04T14:59:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-04T14:59:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,170 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 47aad6a6d662b77a47b7fe8495dd267c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T14:59:02,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=A 2024-12-04T14:59:02,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:02,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=B 2024-12-04T14:59:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:02,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=C 2024-12-04T14:59:02,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041640fe523f43401c831fdd1d8e81682f_47aad6a6d662b77a47b7fe8495dd267c is 50, key is test_row_1/A:col10/1733324341208/Put/seqid=0 2024-12-04T14:59:02,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741950_1126 (size=9714) 2024-12-04T14:59:02,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,213 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041640fe523f43401c831fdd1d8e81682f_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041640fe523f43401c831fdd1d8e81682f_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:02,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/372010f4b8f747cd810afd586d5a77b5, store: [table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:02,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/372010f4b8f747cd810afd586d5a77b5 is 175, key is test_row_1/A:col10/1733324341208/Put/seqid=0 2024-12-04T14:59:02,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741951_1127 (size=22361) 2024-12-04T14:59:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,229 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/372010f4b8f747cd810afd586d5a77b5 2024-12-04T14:59:02,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/f602ddc54f85457fae78dc3b64f137c7 is 50, key is test_row_1/B:col10/1733324341208/Put/seqid=0 2024-12-04T14:59:02,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741952_1128 (size=9657) 2024-12-04T14:59:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,259 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/f602ddc54f85457fae78dc3b64f137c7 2024-12-04T14:59:02,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/2aa7a430ca5e46a5b6c4a51766d6976b is 50, key is test_row_1/C:col10/1733324341208/Put/seqid=0 2024-12-04T14:59:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741953_1129 (size=9657) 2024-12-04T14:59:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,309 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/2aa7a430ca5e46a5b6c4a51766d6976b 2024-12-04T14:59:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/372010f4b8f747cd810afd586d5a77b5 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/372010f4b8f747cd810afd586d5a77b5 2024-12-04T14:59:02,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,321 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/372010f4b8f747cd810afd586d5a77b5, entries=100, sequenceid=60, filesize=21.8 K 2024-12-04T14:59:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/f602ddc54f85457fae78dc3b64f137c7 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/f602ddc54f85457fae78dc3b64f137c7 2024-12-04T14:59:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,329 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/f602ddc54f85457fae78dc3b64f137c7, entries=100, sequenceid=60, filesize=9.4 K 2024-12-04T14:59:02,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/2aa7a430ca5e46a5b6c4a51766d6976b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2aa7a430ca5e46a5b6c4a51766d6976b 2024-12-04T14:59:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,338 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2aa7a430ca5e46a5b6c4a51766d6976b, entries=100, sequenceid=60, filesize=9.4 K 2024-12-04T14:59:02,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,339 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for 47aad6a6d662b77a47b7fe8495dd267c in 169ms, sequenceid=60, compaction requested=true 2024-12-04T14:59:02,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:02,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:02,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-12-04T14:59:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-12-04T14:59:02,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,344 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-04T14:59:02,344 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5800 sec 2024-12-04T14:59:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 1.5900 sec 2024-12-04T14:59:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:02,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47aad6a6d662b77a47b7fe8495dd267c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T14:59:02,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=A 2024-12-04T14:59:02,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:02,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=B 2024-12-04T14:59:02,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:02,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=C 2024-12-04T14:59:02,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:02,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d695c84c51f54487983dab9ade1c4c1c_47aad6a6d662b77a47b7fe8495dd267c is 50, key is test_row_0/A:col10/1733324342416/Put/seqid=0 2024-12-04T14:59:02,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741954_1130 (size=21918) 2024-12-04T14:59:02,469 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:02,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733324342453, max timestamp=1733324342470Num uniques 859; 2024-12-04T14:59:02,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733324342453, max timestamp=1733324342470Num uniques 859; 2024-12-04T14:59:02,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:02,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:02,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=184, cellSize=8424, totalHeapSize=19376, min timestamp=1733324342453, max timestamp=1733324342470Num uniques 859; 2024-12-04T14:59:02,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=8424, suffix heap size=117064,new segment heap size=19376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=184 2024-12-04T14:59:02,482 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d695c84c51f54487983dab9ade1c4c1c_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d695c84c51f54487983dab9ade1c4c1c_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:02,487 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/77a886794d1246afb53c20109b74bf31, store: [table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:02,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/77a886794d1246afb53c20109b74bf31 is 175, key is test_row_0/A:col10/1733324342416/Put/seqid=0 2024-12-04T14:59:02,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:02,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324402500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:02,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:02,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39330 deadline: 1733324402502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:02,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:02,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733324402503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:02,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741955_1131 (size=65323) 2024-12-04T14:59:02,514 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=70, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/77a886794d1246afb53c20109b74bf31 2024-12-04T14:59:02,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:02,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39338 deadline: 1733324402510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:02,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/88b18307c7b349498eae0821e4000a1d is 50, key is test_row_0/B:col10/1733324342416/Put/seqid=0 2024-12-04T14:59:02,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741956_1132 (size=12001) 2024-12-04T14:59:02,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/88b18307c7b349498eae0821e4000a1d 2024-12-04T14:59:02,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/96ea025439bd45b88b452bac6536bc08 is 50, key is test_row_0/C:col10/1733324342416/Put/seqid=0 2024-12-04T14:59:02,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741957_1133 (size=12001) 2024-12-04T14:59:02,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39330 deadline: 1733324402622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:02,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:02,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733324402622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:02,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:02,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:02,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:02,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39338 deadline: 1733324402622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:02,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324402622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:02,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324402831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:02,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733324402831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:02,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39330 deadline: 1733324402831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:02,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39338 deadline: 1733324402831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:02,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:02,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T14:59:02,900 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:02,901 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:59:02,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-12-04T14:59:02,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T14:59:02,902 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:59:02,903 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:59:02,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:59:02,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/96ea025439bd45b88b452bac6536bc08 2024-12-04T14:59:02,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/77a886794d1246afb53c20109b74bf31 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/77a886794d1246afb53c20109b74bf31 2024-12-04T14:59:02,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/77a886794d1246afb53c20109b74bf31, entries=350, sequenceid=70, filesize=63.8 K 2024-12-04T14:59:02,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/88b18307c7b349498eae0821e4000a1d as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/88b18307c7b349498eae0821e4000a1d 2024-12-04T14:59:03,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/88b18307c7b349498eae0821e4000a1d, entries=150, sequenceid=70, filesize=11.7 K 2024-12-04T14:59:03,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/96ea025439bd45b88b452bac6536bc08 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/96ea025439bd45b88b452bac6536bc08 2024-12-04T14:59:03,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/96ea025439bd45b88b452bac6536bc08, entries=150, sequenceid=70, filesize=11.7 K 2024-12-04T14:59:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T14:59:03,011 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.07 KB/136260 for 47aad6a6d662b77a47b7fe8495dd267c in 589ms, sequenceid=70, compaction requested=true 2024-12-04T14:59:03,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:03,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47aad6a6d662b77a47b7fe8495dd267c:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:59:03,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:03,012 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:59:03,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47aad6a6d662b77a47b7fe8495dd267c:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:59:03,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:03,012 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:59:03,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47aad6a6d662b77a47b7fe8495dd267c:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:59:03,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:03,013 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:59:03,013 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 166782 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:59:03,013 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 47aad6a6d662b77a47b7fe8495dd267c/B is initiating minor compaction (all files) 2024-12-04T14:59:03,013 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 47aad6a6d662b77a47b7fe8495dd267c/A is initiating minor compaction (all files) 2024-12-04T14:59:03,013 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47aad6a6d662b77a47b7fe8495dd267c/B in TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:03,013 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47aad6a6d662b77a47b7fe8495dd267c/A in TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:03,013 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/3a505fc8e0834ea4a29b9d555b9f8c10, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/526d2ad5271743e58060da0d08d83c9a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/f602ddc54f85457fae78dc3b64f137c7, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/88b18307c7b349498eae0821e4000a1d] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp, totalSize=44.6 K 2024-12-04T14:59:03,013 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/776ddd523f0748da9eb69c5cd6e202f5, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/050fe7463e73437894b5a69e45082d7f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/372010f4b8f747cd810afd586d5a77b5, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/77a886794d1246afb53c20109b74bf31] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp, totalSize=162.9 K 2024-12-04T14:59:03,013 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:03,014 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/776ddd523f0748da9eb69c5cd6e202f5, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/050fe7463e73437894b5a69e45082d7f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/372010f4b8f747cd810afd586d5a77b5, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/77a886794d1246afb53c20109b74bf31] 2024-12-04T14:59:03,017 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a505fc8e0834ea4a29b9d555b9f8c10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733324340767 2024-12-04T14:59:03,019 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 526d2ad5271743e58060da0d08d83c9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733324340853 2024-12-04T14:59:03,019 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 776ddd523f0748da9eb69c5cd6e202f5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733324340767 2024-12-04T14:59:03,020 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 050fe7463e73437894b5a69e45082d7f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733324340853 2024-12-04T14:59:03,020 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting f602ddc54f85457fae78dc3b64f137c7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733324341204 2024-12-04T14:59:03,020 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 372010f4b8f747cd810afd586d5a77b5, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733324341204 2024-12-04T14:59:03,020 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 88b18307c7b349498eae0821e4000a1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733324342395 2024-12-04T14:59:03,021 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77a886794d1246afb53c20109b74bf31, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733324342394 2024-12-04T14:59:03,032 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:03,035 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204e5a3a245cafd43a1885c85bc6f4b2d48_47aad6a6d662b77a47b7fe8495dd267c store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:03,036 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47aad6a6d662b77a47b7fe8495dd267c#B#compaction#115 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:03,037 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/64d222d9b97e42aebddcf8f84d87b09e is 50, key is test_row_0/B:col10/1733324342416/Put/seqid=0 2024-12-04T14:59:03,040 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204e5a3a245cafd43a1885c85bc6f4b2d48_47aad6a6d662b77a47b7fe8495dd267c, store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:03,040 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e5a3a245cafd43a1885c85bc6f4b2d48_47aad6a6d662b77a47b7fe8495dd267c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:03,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-04T14:59:03,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:03,056 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 47aad6a6d662b77a47b7fe8495dd267c 3/3 column families, dataSize=133.07 KB heapSize=377.76 KB 2024-12-04T14:59:03,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=A 2024-12-04T14:59:03,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:03,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=B 2024-12-04T14:59:03,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:03,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=C 2024-12-04T14:59:03,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:03,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:03,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741958_1134 (size=12139) 2024-12-04T14:59:03,077 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/64d222d9b97e42aebddcf8f84d87b09e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/64d222d9b97e42aebddcf8f84d87b09e 2024-12-04T14:59:03,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741959_1135 (size=4469) 2024-12-04T14:59:03,086 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47aad6a6d662b77a47b7fe8495dd267c#A#compaction#114 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:03,087 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/4e27dd90e2234a4d9e020d1ad486b53d is 175, key is test_row_0/A:col10/1733324342416/Put/seqid=0 2024-12-04T14:59:03,091 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47aad6a6d662b77a47b7fe8495dd267c/B of 47aad6a6d662b77a47b7fe8495dd267c into 64d222d9b97e42aebddcf8f84d87b09e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:03,091 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:03,091 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., storeName=47aad6a6d662b77a47b7fe8495dd267c/B, priority=12, startTime=1733324343012; duration=0sec 2024-12-04T14:59:03,091 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:03,091 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47aad6a6d662b77a47b7fe8495dd267c:B 2024-12-04T14:59:03,092 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:59:03,093 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:59:03,094 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 47aad6a6d662b77a47b7fe8495dd267c/C is initiating minor compaction (all files) 2024-12-04T14:59:03,094 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47aad6a6d662b77a47b7fe8495dd267c/C in TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:03,094 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/c01e14c081c04805bf4ab65718927b20, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/13fbad04d2ef4b2cbf9d040695385791, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2aa7a430ca5e46a5b6c4a51766d6976b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/96ea025439bd45b88b452bac6536bc08] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp, totalSize=44.6 K 2024-12-04T14:59:03,095 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting c01e14c081c04805bf4ab65718927b20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733324340767 2024-12-04T14:59:03,095 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 13fbad04d2ef4b2cbf9d040695385791, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733324340853 2024-12-04T14:59:03,096 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aa7a430ca5e46a5b6c4a51766d6976b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733324341204 2024-12-04T14:59:03,097 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 96ea025439bd45b88b452bac6536bc08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733324342395 2024-12-04T14:59:03,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:03,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324403094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:03,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:03,141 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47aad6a6d662b77a47b7fe8495dd267c#C#compaction#116 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:03,141 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/a0970d07d1304bd1bf43a165ee569ea2 is 50, key is test_row_0/C:col10/1733324342416/Put/seqid=0 2024-12-04T14:59:03,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120453922ade24b742b5b21e2d6b9ca91a0f_47aad6a6d662b77a47b7fe8495dd267c is 50, key is test_row_0/A:col10/1733324342500/Put/seqid=0 2024-12-04T14:59:03,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:03,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324403150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:03,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:03,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733324403150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:03,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:03,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39338 deadline: 1733324403150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:03,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:03,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39330 deadline: 1733324403151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:03,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:03,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741960_1136 (size=31093) 2024-12-04T14:59:03,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741961_1137 (size=12139) 2024-12-04T14:59:03,198 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/a0970d07d1304bd1bf43a165ee569ea2 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/a0970d07d1304bd1bf43a165ee569ea2 2024-12-04T14:59:03,208 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47aad6a6d662b77a47b7fe8495dd267c/C of 47aad6a6d662b77a47b7fe8495dd267c into a0970d07d1304bd1bf43a165ee569ea2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:03,208 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:03,208 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., storeName=47aad6a6d662b77a47b7fe8495dd267c/C, priority=12, startTime=1733324343012; duration=0sec 2024-12-04T14:59:03,208 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:03,208 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47aad6a6d662b77a47b7fe8495dd267c:C 2024-12-04T14:59:03,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741962_1138 (size=12154) 2024-12-04T14:59:03,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:03,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324403210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:03,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:03,220 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120453922ade24b742b5b21e2d6b9ca91a0f_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120453922ade24b742b5b21e2d6b9ca91a0f_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T14:59:03,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/6df324351c3945368c42193cc07f10f5, store: [table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:03,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/6df324351c3945368c42193cc07f10f5 is 175, key is test_row_0/A:col10/1733324342500/Put/seqid=0 2024-12-04T14:59:03,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741963_1139 (size=30955) 2024-12-04T14:59:03,239 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=106, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/6df324351c3945368c42193cc07f10f5 2024-12-04T14:59:03,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/4873b4aa1ed649a58debcb7e16adc1ed is 50, key is test_row_0/B:col10/1733324342500/Put/seqid=0 2024-12-04T14:59:03,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741964_1140 (size=12001) 2024-12-04T14:59:03,271 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/4873b4aa1ed649a58debcb7e16adc1ed 2024-12-04T14:59:03,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/52ef8a602a664cbcac6d791606e3e685 is 50, key is test_row_0/C:col10/1733324342500/Put/seqid=0 2024-12-04T14:59:03,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741965_1141 (size=12001) 2024-12-04T14:59:03,290 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.61 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/52ef8a602a664cbcac6d791606e3e685 2024-12-04T14:59:03,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/6df324351c3945368c42193cc07f10f5 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6df324351c3945368c42193cc07f10f5 2024-12-04T14:59:03,306 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6df324351c3945368c42193cc07f10f5, entries=150, sequenceid=106, filesize=30.2 K 2024-12-04T14:59:03,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/4873b4aa1ed649a58debcb7e16adc1ed as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/4873b4aa1ed649a58debcb7e16adc1ed 2024-12-04T14:59:03,314 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/4873b4aa1ed649a58debcb7e16adc1ed, entries=150, sequenceid=106, filesize=11.7 K 2024-12-04T14:59:03,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/52ef8a602a664cbcac6d791606e3e685 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/52ef8a602a664cbcac6d791606e3e685 2024-12-04T14:59:03,324 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/52ef8a602a664cbcac6d791606e3e685, entries=150, sequenceid=106, filesize=11.7 K 2024-12-04T14:59:03,325 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~133.07 KB/136260, heapSize ~377.71 KB/386776, currentSize=46.96 KB/48090 for 47aad6a6d662b77a47b7fe8495dd267c in 269ms, sequenceid=106, compaction requested=false 2024-12-04T14:59:03,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:03,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:03,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-12-04T14:59:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-12-04T14:59:03,328 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-04T14:59:03,328 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 423 msec 2024-12-04T14:59:03,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 427 msec 2024-12-04T14:59:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:03,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47aad6a6d662b77a47b7fe8495dd267c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T14:59:03,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=A 2024-12-04T14:59:03,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:03,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=B 2024-12-04T14:59:03,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:03,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=C 2024-12-04T14:59:03,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:03,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412046cc1a7d8fa4d454b9ca520db6e896caf_47aad6a6d662b77a47b7fe8495dd267c is 50, key is test_row_0/A:col10/1733324343082/Put/seqid=0 2024-12-04T14:59:03,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741966_1142 (size=14594) 2024-12-04T14:59:03,489 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:03,497 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412046cc1a7d8fa4d454b9ca520db6e896caf_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046cc1a7d8fa4d454b9ca520db6e896caf_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:03,499 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/2ca2263f6681429a8b6a208ec4cb7f9c, store: [table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:03,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/2ca2263f6681429a8b6a208ec4cb7f9c is 175, key is test_row_0/A:col10/1733324343082/Put/seqid=0 2024-12-04T14:59:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T14:59:03,531 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:03,531 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:59:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-04T14:59:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-04T14:59:03,534 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:59:03,535 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:59:03,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:59:03,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741967_1143 (size=39549) 2024-12-04T14:59:03,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324343463, max timestamp=1733324343534Num uniques 859; 2024-12-04T14:59:03,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324343463, max timestamp=1733324343534Num uniques 859; 2024-12-04T14:59:03,537 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/2ca2263f6681429a8b6a208ec4cb7f9c 2024-12-04T14:59:03,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:03,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:03,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324343463, max timestamp=1733324343534Num uniques 859; 2024-12-04T14:59:03,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:03,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/662b1f4bd7274c0f8940d5da872755f8 is 50, key is test_row_0/B:col10/1733324343082/Put/seqid=0 2024-12-04T14:59:03,587 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/4e27dd90e2234a4d9e020d1ad486b53d as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/4e27dd90e2234a4d9e020d1ad486b53d 2024-12-04T14:59:03,601 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47aad6a6d662b77a47b7fe8495dd267c/A of 47aad6a6d662b77a47b7fe8495dd267c into 4e27dd90e2234a4d9e020d1ad486b53d(size=30.4 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:03,601 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:03,601 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., storeName=47aad6a6d662b77a47b7fe8495dd267c/A, priority=12, startTime=1733324343011; duration=0sec 2024-12-04T14:59:03,601 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:03,601 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47aad6a6d662b77a47b7fe8495dd267c:A 2024-12-04T14:59:03,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741968_1144 (size=12001) 2024-12-04T14:59:03,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/662b1f4bd7274c0f8940d5da872755f8 2024-12-04T14:59:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-04T14:59:03,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/d8b85c363c904dcc8eb9e61761abb8c0 is 50, key is test_row_0/C:col10/1733324343082/Put/seqid=0 2024-12-04T14:59:03,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39338 deadline: 1733324403661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:03,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:03,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:03,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39330 deadline: 1733324403661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733324403662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:03,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:03,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:03,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324403662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:03,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324403667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:03,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:03,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741969_1145 (size=12001) 2024-12-04T14:59:03,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/d8b85c363c904dcc8eb9e61761abb8c0 2024-12-04T14:59:03,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-04T14:59:03,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:03,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:03,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:03,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:03,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:03,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/2ca2263f6681429a8b6a208ec4cb7f9c as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/2ca2263f6681429a8b6a208ec4cb7f9c 2024-12-04T14:59:03,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/2ca2263f6681429a8b6a208ec4cb7f9c, entries=200, sequenceid=119, filesize=38.6 K 2024-12-04T14:59:03,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/662b1f4bd7274c0f8940d5da872755f8 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/662b1f4bd7274c0f8940d5da872755f8 2024-12-04T14:59:03,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/662b1f4bd7274c0f8940d5da872755f8, entries=150, sequenceid=119, filesize=11.7 K 2024-12-04T14:59:03,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/d8b85c363c904dcc8eb9e61761abb8c0 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/d8b85c363c904dcc8eb9e61761abb8c0 2024-12-04T14:59:03,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/d8b85c363c904dcc8eb9e61761abb8c0, entries=150, sequenceid=119, filesize=11.7 K 2024-12-04T14:59:03,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 47aad6a6d662b77a47b7fe8495dd267c in 309ms, sequenceid=119, compaction requested=true 2024-12-04T14:59:03,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:03,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47aad6a6d662b77a47b7fe8495dd267c:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:59:03,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:03,731 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:03,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47aad6a6d662b77a47b7fe8495dd267c:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:59:03,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:03,731 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:03,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47aad6a6d662b77a47b7fe8495dd267c:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:59:03,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:03,733 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101597 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:03,733 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 47aad6a6d662b77a47b7fe8495dd267c/A is initiating minor compaction (all files) 2024-12-04T14:59:03,733 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47aad6a6d662b77a47b7fe8495dd267c/A in TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:03,733 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/4e27dd90e2234a4d9e020d1ad486b53d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6df324351c3945368c42193cc07f10f5, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/2ca2263f6681429a8b6a208ec4cb7f9c] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp, totalSize=99.2 K 2024-12-04T14:59:03,733 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:03,733 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/4e27dd90e2234a4d9e020d1ad486b53d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6df324351c3945368c42193cc07f10f5, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/2ca2263f6681429a8b6a208ec4cb7f9c] 2024-12-04T14:59:03,734 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:03,734 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 47aad6a6d662b77a47b7fe8495dd267c/B is initiating minor compaction (all files) 2024-12-04T14:59:03,734 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47aad6a6d662b77a47b7fe8495dd267c/B in TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:03,734 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/64d222d9b97e42aebddcf8f84d87b09e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/4873b4aa1ed649a58debcb7e16adc1ed, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/662b1f4bd7274c0f8940d5da872755f8] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp, totalSize=35.3 K 2024-12-04T14:59:03,734 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 64d222d9b97e42aebddcf8f84d87b09e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733324342395 2024-12-04T14:59:03,735 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e27dd90e2234a4d9e020d1ad486b53d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733324342395 2024-12-04T14:59:03,735 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 4873b4aa1ed649a58debcb7e16adc1ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733324342500 2024-12-04T14:59:03,735 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6df324351c3945368c42193cc07f10f5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733324342500 2024-12-04T14:59:03,736 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 662b1f4bd7274c0f8940d5da872755f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733324343074 2024-12-04T14:59:03,736 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ca2263f6681429a8b6a208ec4cb7f9c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733324343074 2024-12-04T14:59:03,761 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47aad6a6d662b77a47b7fe8495dd267c#B#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:03,761 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/e3e053773bbe4120a5f7684be7a2f99c is 50, key is test_row_0/B:col10/1733324343082/Put/seqid=0 2024-12-04T14:59:03,767 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:03,775 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412046f4c6340e114458bb31b536bc4b450f2_47aad6a6d662b77a47b7fe8495dd267c store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:03,777 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412046f4c6340e114458bb31b536bc4b450f2_47aad6a6d662b77a47b7fe8495dd267c, store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:03,777 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412046f4c6340e114458bb31b536bc4b450f2_47aad6a6d662b77a47b7fe8495dd267c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:03,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741970_1146 (size=12241) 2024-12-04T14:59:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:03,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47aad6a6d662b77a47b7fe8495dd267c 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-04T14:59:03,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=A 2024-12-04T14:59:03,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:03,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=B 2024-12-04T14:59:03,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:03,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=C 2024-12-04T14:59:03,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:03,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741971_1147 (size=4469) 2024-12-04T14:59:03,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120487c303a5634c4af5bec6485bdb00bb95_47aad6a6d662b77a47b7fe8495dd267c is 50, key is test_row_0/A:col10/1733324343784/Put/seqid=0 2024-12-04T14:59:03,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:03,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324403836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:03,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:03,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741972_1148 (size=14794) 2024-12-04T14:59:03,848 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:03,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-04T14:59:03,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-04T14:59:03,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:03,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:03,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:03,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:03,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:03,855 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120487c303a5634c4af5bec6485bdb00bb95_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120487c303a5634c4af5bec6485bdb00bb95_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:03,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:03,860 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/7d6bec5c0c3f41fbae4059e42b21971e, store: [table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:03,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/7d6bec5c0c3f41fbae4059e42b21971e is 175, key is test_row_0/A:col10/1733324343784/Put/seqid=0 2024-12-04T14:59:03,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741973_1149 (size=39749) 2024-12-04T14:59:03,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324403950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:04,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-04T14:59:04,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:04,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:04,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:04,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:04,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:04,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:04,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-04T14:59:04,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:04,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-04T14:59:04,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:04,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:04,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:04,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:04,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:04,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:04,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324404161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:04,188 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/e3e053773bbe4120a5f7684be7a2f99c as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/e3e053773bbe4120a5f7684be7a2f99c 2024-12-04T14:59:04,191 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47aad6a6d662b77a47b7fe8495dd267c#A#compaction#124 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:04,192 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/6a70daf6658c4896a813d394c69d5af8 is 175, key is test_row_0/A:col10/1733324343082/Put/seqid=0 2024-12-04T14:59:04,195 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47aad6a6d662b77a47b7fe8495dd267c/B of 47aad6a6d662b77a47b7fe8495dd267c into e3e053773bbe4120a5f7684be7a2f99c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:04,195 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:04,195 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., storeName=47aad6a6d662b77a47b7fe8495dd267c/B, priority=13, startTime=1733324343731; duration=0sec 2024-12-04T14:59:04,195 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:04,195 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47aad6a6d662b77a47b7fe8495dd267c:B 2024-12-04T14:59:04,195 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:04,197 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:04,197 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 47aad6a6d662b77a47b7fe8495dd267c/C is initiating minor compaction (all files) 2024-12-04T14:59:04,197 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47aad6a6d662b77a47b7fe8495dd267c/C in TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:04,197 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/a0970d07d1304bd1bf43a165ee569ea2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/52ef8a602a664cbcac6d791606e3e685, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/d8b85c363c904dcc8eb9e61761abb8c0] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp, totalSize=35.3 K 2024-12-04T14:59:04,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741974_1150 (size=31195) 2024-12-04T14:59:04,199 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting a0970d07d1304bd1bf43a165ee569ea2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733324342395 2024-12-04T14:59:04,200 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 52ef8a602a664cbcac6d791606e3e685, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733324342500 2024-12-04T14:59:04,200 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting d8b85c363c904dcc8eb9e61761abb8c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733324343074 2024-12-04T14:59:04,205 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47aad6a6d662b77a47b7fe8495dd267c#C#compaction#126 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:04,205 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/735d255f8a594061a3329ffbdc5f0f1e is 50, key is test_row_0/C:col10/1733324343082/Put/seqid=0 2024-12-04T14:59:04,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741975_1151 (size=12241) 2024-12-04T14:59:04,216 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/735d255f8a594061a3329ffbdc5f0f1e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/735d255f8a594061a3329ffbdc5f0f1e 2024-12-04T14:59:04,224 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47aad6a6d662b77a47b7fe8495dd267c/C of 47aad6a6d662b77a47b7fe8495dd267c into 735d255f8a594061a3329ffbdc5f0f1e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:04,225 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:04,225 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., storeName=47aad6a6d662b77a47b7fe8495dd267c/C, priority=13, startTime=1733324343731; duration=0sec 2024-12-04T14:59:04,225 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:04,225 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47aad6a6d662b77a47b7fe8495dd267c:C 2024-12-04T14:59:04,291 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/7d6bec5c0c3f41fbae4059e42b21971e 2024-12-04T14:59:04,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/b5dd80edf3b44c9e87f99720db9b3137 is 50, key is test_row_0/B:col10/1733324343784/Put/seqid=0 2024-12-04T14:59:04,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741976_1152 (size=12151) 2024-12-04T14:59:04,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/b5dd80edf3b44c9e87f99720db9b3137 2024-12-04T14:59:04,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-04T14:59:04,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:04,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:04,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:04,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:04,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:04,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:04,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/fbf0e52a597b4c169ecd6ef1931ccb85 is 50, key is test_row_0/C:col10/1733324343784/Put/seqid=0 2024-12-04T14:59:04,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741977_1153 (size=12151) 2024-12-04T14:59:04,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/fbf0e52a597b4c169ecd6ef1931ccb85 2024-12-04T14:59:04,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/7d6bec5c0c3f41fbae4059e42b21971e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/7d6bec5c0c3f41fbae4059e42b21971e 2024-12-04T14:59:04,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/7d6bec5c0c3f41fbae4059e42b21971e, entries=200, sequenceid=156, filesize=38.8 K 2024-12-04T14:59:04,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/b5dd80edf3b44c9e87f99720db9b3137 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/b5dd80edf3b44c9e87f99720db9b3137 2024-12-04T14:59:04,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/b5dd80edf3b44c9e87f99720db9b3137, entries=150, sequenceid=156, filesize=11.9 K 2024-12-04T14:59:04,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/fbf0e52a597b4c169ecd6ef1931ccb85 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/fbf0e52a597b4c169ecd6ef1931ccb85 2024-12-04T14:59:04,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/fbf0e52a597b4c169ecd6ef1931ccb85, entries=150, sequenceid=156, filesize=11.9 K 2024-12-04T14:59:04,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 47aad6a6d662b77a47b7fe8495dd267c in 601ms, sequenceid=156, compaction requested=false 2024-12-04T14:59:04,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:04,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-04T14:59:04,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:04,467 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 47aad6a6d662b77a47b7fe8495dd267c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T14:59:04,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=A 2024-12-04T14:59:04,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:04,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=B 2024-12-04T14:59:04,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:04,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=C 2024-12-04T14:59:04,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:04,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:04,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:04,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204cc7975a938d04312b225f36a6d723775_47aad6a6d662b77a47b7fe8495dd267c is 50, key is test_row_0/A:col10/1733324343834/Put/seqid=0 2024-12-04T14:59:04,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741978_1154 (size=12304) 2024-12-04T14:59:04,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324344540, max timestamp=1733324344573Num uniques 859; 2024-12-04T14:59:04,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324344540, max timestamp=1733324344573Num uniques 859; 2024-12-04T14:59:04,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:59:04,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:59:04,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324344540, max timestamp=1733324344573Num uniques 859; 2024-12-04T14:59:04,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:59:04,615 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/6a70daf6658c4896a813d394c69d5af8 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6a70daf6658c4896a813d394c69d5af8 2024-12-04T14:59:04,631 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47aad6a6d662b77a47b7fe8495dd267c/A of 47aad6a6d662b77a47b7fe8495dd267c into 6a70daf6658c4896a813d394c69d5af8(size=30.5 K), total size for store is 69.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:04,631 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:04,631 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., storeName=47aad6a6d662b77a47b7fe8495dd267c/A, priority=13, startTime=1733324343731; duration=0sec 2024-12-04T14:59:04,631 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:04,631 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47aad6a6d662b77a47b7fe8495dd267c:A 2024-12-04T14:59:04,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-04T14:59:04,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:04,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324404678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:04,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:04,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39338 deadline: 1733324404680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:04,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39330 deadline: 1733324404680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:04,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733324404680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:04,683 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 47aad6a6d662b77a47b7fe8495dd267c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:04,683 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 47aad6a6d662b77a47b7fe8495dd267c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:04,684 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_0' in 47aad6a6d662b77a47b7fe8495dd267c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:04,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324404681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:04,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:04,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324404790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:04,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:04,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324404790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:04,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:04,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:04,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:04,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:04,920 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204cc7975a938d04312b225f36a6d723775_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204cc7975a938d04312b225f36a6d723775_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:04,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/14af9a4170a5454dac525e8f3e5157a2, store: [table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:04,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/14af9a4170a5454dac525e8f3e5157a2 is 175, key is test_row_0/A:col10/1733324343834/Put/seqid=0 2024-12-04T14:59:04,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741979_1155 (size=31105) 2024-12-04T14:59:04,954 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/14af9a4170a5454dac525e8f3e5157a2 2024-12-04T14:59:04,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/2c9394aae445466bbf02ecdc6bcb05b2 is 50, key is test_row_0/B:col10/1733324343834/Put/seqid=0 2024-12-04T14:59:04,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741980_1156 (size=12151) 2024-12-04T14:59:05,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:05,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324405000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:05,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:05,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324405000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:05,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324405310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:05,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324405310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:05,390 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/2c9394aae445466bbf02ecdc6bcb05b2 2024-12-04T14:59:05,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/257cc7c7d4ff46d1b67d59edce8025bf is 50, key is test_row_0/C:col10/1733324343834/Put/seqid=0 2024-12-04T14:59:05,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741981_1157 (size=12151) 2024-12-04T14:59:05,473 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/257cc7c7d4ff46d1b67d59edce8025bf 2024-12-04T14:59:05,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/14af9a4170a5454dac525e8f3e5157a2 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/14af9a4170a5454dac525e8f3e5157a2 2024-12-04T14:59:05,491 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/14af9a4170a5454dac525e8f3e5157a2, entries=150, sequenceid=168, filesize=30.4 K 2024-12-04T14:59:05,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/2c9394aae445466bbf02ecdc6bcb05b2 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/2c9394aae445466bbf02ecdc6bcb05b2 2024-12-04T14:59:05,502 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/2c9394aae445466bbf02ecdc6bcb05b2, entries=150, sequenceid=168, filesize=11.9 K 2024-12-04T14:59:05,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/257cc7c7d4ff46d1b67d59edce8025bf as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/257cc7c7d4ff46d1b67d59edce8025bf 2024-12-04T14:59:05,510 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/257cc7c7d4ff46d1b67d59edce8025bf, entries=150, sequenceid=168, filesize=11.9 K 2024-12-04T14:59:05,512 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 47aad6a6d662b77a47b7fe8495dd267c in 1045ms, sequenceid=168, compaction requested=true 2024-12-04T14:59:05,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:05,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:05,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-04T14:59:05,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-04T14:59:05,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-04T14:59:05,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9790 sec 2024-12-04T14:59:05,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.9860 sec 2024-12-04T14:59:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-04T14:59:05,683 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:05,684 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:59:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-04T14:59:05,685 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:59:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T14:59:05,686 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:59:05,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:59:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T14:59:05,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7682, totalHeapSize=17712, min timestamp=1733324344666, max timestamp=1733324345832Num uniques 1018; 2024-12-04T14:59:05,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46612, new segment data size=7682, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T14:59:05,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733324344666, max timestamp=1733324345832Num uniques 1018; 2024-12-04T14:59:05,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T14:59:05,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7680, totalHeapSize=17712, min timestamp=1733324344666, max timestamp=1733324345832Num uniques 1018; 2024-12-04T14:59:05,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46610, new segment data size=7680, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T14:59:05,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T14:59:05,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:05,839 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 47aad6a6d662b77a47b7fe8495dd267c 3/3 column families, dataSize=26.84 KB heapSize=65.39 KB 2024-12-04T14:59:05,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=A 2024-12-04T14:59:05,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:05,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=B 2024-12-04T14:59:05,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:05,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=C 2024-12-04T14:59:05,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:05,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204218aaac014dc4affbac5b1e7124fad05_47aad6a6d662b77a47b7fe8495dd267c is 50, key is test_row_0/A:col10/1733324345830/Put/seqid=0 2024-12-04T14:59:05,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:05,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:05,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741982_1158 (size=17284) 2024-12-04T14:59:05,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:05,891 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204218aaac014dc4affbac5b1e7124fad05_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204218aaac014dc4affbac5b1e7124fad05_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:05,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/6b2f76428d48496eb053765985ad9cf3, store: [table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:05,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/6b2f76428d48496eb053765985ad9cf3 is 175, key is test_row_0/A:col10/1733324345830/Put/seqid=0 2024-12-04T14:59:05,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733324345868, max timestamp=1733324345897Num uniques 859; 2024-12-04T14:59:05,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T14:59:05,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733324345868, max timestamp=1733324345897Num uniques 859; 2024-12-04T14:59:05,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T14:59:05,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733324345868, max timestamp=1733324345897Num uniques 859; 2024-12-04T14:59:05,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T14:59:05,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741983_1159 (size=48389) 2024-12-04T14:59:06,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733324345942, max timestamp=1733324345997Num uniques 1118; 2024-12-04T14:59:06,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-04T14:59:06,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733324345942, max timestamp=1733324345997Num uniques 1118; 2024-12-04T14:59:06,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733324345942, max timestamp=1733324345997Num uniques 1118; 2024-12-04T14:59:06,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-04T14:59:06,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-04T14:59:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T14:59:06,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733324345997, max timestamp=1733324346091Num uniques 1027; 2024-12-04T14:59:06,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733324345997, max timestamp=1733324346091Num uniques 1027; 2024-12-04T14:59:06,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-12-04T14:59:06,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-12-04T14:59:06,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733324345997, max timestamp=1733324346091Num uniques 1027; 2024-12-04T14:59:06,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-12-04T14:59:06,179 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T14:59:06,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733324346131, max timestamp=1733324346227Num uniques 1136; 2024-12-04T14:59:06,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=15386, suffix heap size=146112,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=336 2024-12-04T14:59:06,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733324346131, max timestamp=1733324346227Num uniques 1136; 2024-12-04T14:59:06,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=15386, suffix heap size=146112,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=336 2024-12-04T14:59:06,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733324346131, max timestamp=1733324346227Num uniques 1136; 2024-12-04T14:59:06,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=15386, suffix heap size=146112,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=336 2024-12-04T14:59:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T14:59:06,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324406316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:06,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:06,333 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/6b2f76428d48496eb053765985ad9cf3 2024-12-04T14:59:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324406318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:06,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/969bfe2fb47d43bea8dcb3c50d75f7eb is 50, key is test_row_0/B:col10/1733324345830/Put/seqid=0 2024-12-04T14:59:06,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741984_1160 (size=12151) 2024-12-04T14:59:06,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:06,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324406450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324406450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:06,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:06,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:06,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:06,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324406660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:06,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:06,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324406660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:06,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:06,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:06,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733324406690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:06,693 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_0' in 47aad6a6d662b77a47b7fe8495dd267c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:06,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:06,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39330 deadline: 1733324406700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:06,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39338 deadline: 1733324406700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:06,703 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 47aad6a6d662b77a47b7fe8495dd267c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:06,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:06,704 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 47aad6a6d662b77a47b7fe8495dd267c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:06,813 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/969bfe2fb47d43bea8dcb3c50d75f7eb 2024-12-04T14:59:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T14:59:06,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/bee8b8cbe1f3491e89047ea67cf2f4ee is 50, key is test_row_0/C:col10/1733324345830/Put/seqid=0 2024-12-04T14:59:06,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741985_1161 (size=12151) 2024-12-04T14:59:06,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324406970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:06,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324406970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:06,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:06,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:07,232 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/bee8b8cbe1f3491e89047ea67cf2f4ee 2024-12-04T14:59:07,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/6b2f76428d48496eb053765985ad9cf3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6b2f76428d48496eb053765985ad9cf3 2024-12-04T14:59:07,244 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6b2f76428d48496eb053765985ad9cf3, entries=250, sequenceid=208, filesize=47.3 K 2024-12-04T14:59:07,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/969bfe2fb47d43bea8dcb3c50d75f7eb as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/969bfe2fb47d43bea8dcb3c50d75f7eb 2024-12-04T14:59:07,267 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/969bfe2fb47d43bea8dcb3c50d75f7eb, entries=150, sequenceid=208, filesize=11.9 K 2024-12-04T14:59:07,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/bee8b8cbe1f3491e89047ea67cf2f4ee as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/bee8b8cbe1f3491e89047ea67cf2f4ee 2024-12-04T14:59:07,275 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/bee8b8cbe1f3491e89047ea67cf2f4ee, entries=150, sequenceid=208, filesize=11.9 K 2024-12-04T14:59:07,276 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=154.31 KB/158010 for 47aad6a6d662b77a47b7fe8495dd267c in 1437ms, sequenceid=208, compaction requested=true 2024-12-04T14:59:07,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:07,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:07,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-04T14:59:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-04T14:59:07,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-04T14:59:07,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5920 sec 2024-12-04T14:59:07,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.5960 sec 2024-12-04T14:59:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:07,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47aad6a6d662b77a47b7fe8495dd267c 3/3 column families, dataSize=161.02 KB heapSize=448.78 KB 2024-12-04T14:59:07,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=A 2024-12-04T14:59:07,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T14:59:07,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=B 2024-12-04T14:59:07,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T14:59:07,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=C 2024-12-04T14:59:07,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T14:59:07,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324407489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:07,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324407493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:07,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204aed5bca425a94a309f81f9328a151b2a_47aad6a6d662b77a47b7fe8495dd267c is 50, key is test_row_0/A:col10/1733324346314/Put/seqid=0 2024-12-04T14:59:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:07,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741986_1162 (size=14994) 2024-12-04T14:59:07,503 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:07,507 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204aed5bca425a94a309f81f9328a151b2a_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204aed5bca425a94a309f81f9328a151b2a_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:07,507 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/1f4c20ba607d4d6ca184112392497770, store: [table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:07,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/1f4c20ba607d4d6ca184112392497770 is 175, key is test_row_0/A:col10/1733324346314/Put/seqid=0 2024-12-04T14:59:07,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741987_1163 (size=39949) 2024-12-04T14:59:07,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324407600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:07,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:07,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:07,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:07,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324407600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:07,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:07,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:07,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:07,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:07,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324407810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:07,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:07,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:07,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:07,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324407810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:07,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T14:59:07,831 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:07,831 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:59:07,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-04T14:59:07,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T14:59:07,833 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:59:07,833 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:59:07,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:59:07,920 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/1f4c20ba607d4d6ca184112392497770 2024-12-04T14:59:07,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/b8c1fd0403574008b133fbaf09a2ac20 is 50, key is test_row_0/B:col10/1733324346314/Put/seqid=0 2024-12-04T14:59:07,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T14:59:07,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741988_1164 (size=12301) 2024-12-04T14:59:07,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/b8c1fd0403574008b133fbaf09a2ac20 2024-12-04T14:59:07,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/d7db5ce70800493989ecb047663e21d3 is 50, key is test_row_0/C:col10/1733324346314/Put/seqid=0 2024-12-04T14:59:07,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741989_1165 (size=12301) 2024-12-04T14:59:07,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/d7db5ce70800493989ecb047663e21d3 2024-12-04T14:59:07,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/1f4c20ba607d4d6ca184112392497770 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/1f4c20ba607d4d6ca184112392497770 2024-12-04T14:59:07,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T14:59:07,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:07,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:07,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:07,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:07,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:07,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:08,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/1f4c20ba607d4d6ca184112392497770, entries=200, sequenceid=297, filesize=39.0 K 2024-12-04T14:59:08,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/b8c1fd0403574008b133fbaf09a2ac20 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/b8c1fd0403574008b133fbaf09a2ac20 2024-12-04T14:59:08,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/b8c1fd0403574008b133fbaf09a2ac20, entries=150, sequenceid=297, filesize=12.0 K 2024-12-04T14:59:08,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/d7db5ce70800493989ecb047663e21d3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/d7db5ce70800493989ecb047663e21d3 2024-12-04T14:59:08,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/d7db5ce70800493989ecb047663e21d3, entries=150, sequenceid=297, filesize=12.0 K 2024-12-04T14:59:08,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~448.73 KB/459504, currentSize=26.84 KB/27480 for 47aad6a6d662b77a47b7fe8495dd267c in 544ms, sequenceid=297, compaction requested=true 2024-12-04T14:59:08,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:08,027 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T14:59:08,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47aad6a6d662b77a47b7fe8495dd267c:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:59:08,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:08,028 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T14:59:08,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47aad6a6d662b77a47b7fe8495dd267c:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:59:08,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:08,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47aad6a6d662b77a47b7fe8495dd267c:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:59:08,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:08,029 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 190387 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T14:59:08,029 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 47aad6a6d662b77a47b7fe8495dd267c/A is initiating minor compaction (all files) 2024-12-04T14:59:08,029 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47aad6a6d662b77a47b7fe8495dd267c/A in TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:08,029 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6a70daf6658c4896a813d394c69d5af8, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/7d6bec5c0c3f41fbae4059e42b21971e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/14af9a4170a5454dac525e8f3e5157a2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6b2f76428d48496eb053765985ad9cf3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/1f4c20ba607d4d6ca184112392497770] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp, totalSize=185.9 K 2024-12-04T14:59:08,029 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:08,030 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6a70daf6658c4896a813d394c69d5af8, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/7d6bec5c0c3f41fbae4059e42b21971e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/14af9a4170a5454dac525e8f3e5157a2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6b2f76428d48496eb053765985ad9cf3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/1f4c20ba607d4d6ca184112392497770] 2024-12-04T14:59:08,030 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a70daf6658c4896a813d394c69d5af8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733324343074 2024-12-04T14:59:08,031 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60995 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T14:59:08,031 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d6bec5c0c3f41fbae4059e42b21971e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733324343624 2024-12-04T14:59:08,031 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 47aad6a6d662b77a47b7fe8495dd267c/B is initiating minor compaction (all files) 2024-12-04T14:59:08,031 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47aad6a6d662b77a47b7fe8495dd267c/B in TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:08,032 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/e3e053773bbe4120a5f7684be7a2f99c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/b5dd80edf3b44c9e87f99720db9b3137, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/2c9394aae445466bbf02ecdc6bcb05b2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/969bfe2fb47d43bea8dcb3c50d75f7eb, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/b8c1fd0403574008b133fbaf09a2ac20] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp, totalSize=59.6 K 2024-12-04T14:59:08,032 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting e3e053773bbe4120a5f7684be7a2f99c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733324343074 2024-12-04T14:59:08,032 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14af9a4170a5454dac525e8f3e5157a2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733324343798 2024-12-04T14:59:08,033 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting b5dd80edf3b44c9e87f99720db9b3137, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733324343624 2024-12-04T14:59:08,033 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b2f76428d48496eb053765985ad9cf3, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733324344666 2024-12-04T14:59:08,033 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c9394aae445466bbf02ecdc6bcb05b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733324343798 2024-12-04T14:59:08,033 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f4c20ba607d4d6ca184112392497770, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733324346303 2024-12-04T14:59:08,033 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 969bfe2fb47d43bea8dcb3c50d75f7eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733324344679 2024-12-04T14:59:08,033 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting b8c1fd0403574008b133fbaf09a2ac20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733324346303 2024-12-04T14:59:08,047 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47aad6a6d662b77a47b7fe8495dd267c#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:08,048 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:08,048 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/446c89a74831498297bf7d8565114c94 is 50, key is test_row_0/B:col10/1733324346314/Put/seqid=0 2024-12-04T14:59:08,063 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204a6647ac394504d2587fe4e8abea8cfb6_47aad6a6d662b77a47b7fe8495dd267c store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:08,066 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204a6647ac394504d2587fe4e8abea8cfb6_47aad6a6d662b77a47b7fe8495dd267c, store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:08,066 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a6647ac394504d2587fe4e8abea8cfb6_47aad6a6d662b77a47b7fe8495dd267c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:08,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741990_1166 (size=12711) 2024-12-04T14:59:08,087 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/446c89a74831498297bf7d8565114c94 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/446c89a74831498297bf7d8565114c94 2024-12-04T14:59:08,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741991_1167 (size=4469) 2024-12-04T14:59:08,093 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47aad6a6d662b77a47b7fe8495dd267c#A#compaction#139 average throughput is 0.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:08,094 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/d9469c995c084596948281fa62b27df9 is 175, key is test_row_0/A:col10/1733324346314/Put/seqid=0 2024-12-04T14:59:08,115 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 47aad6a6d662b77a47b7fe8495dd267c/B of 47aad6a6d662b77a47b7fe8495dd267c into 446c89a74831498297bf7d8565114c94(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:08,115 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:08,115 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., storeName=47aad6a6d662b77a47b7fe8495dd267c/B, priority=11, startTime=1733324348028; duration=0sec 2024-12-04T14:59:08,115 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:08,115 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47aad6a6d662b77a47b7fe8495dd267c:B 2024-12-04T14:59:08,115 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T14:59:08,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741992_1168 (size=31665) 2024-12-04T14:59:08,118 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60995 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T14:59:08,118 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 47aad6a6d662b77a47b7fe8495dd267c/C is initiating minor compaction (all files) 2024-12-04T14:59:08,118 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47aad6a6d662b77a47b7fe8495dd267c/C in TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:08,119 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/735d255f8a594061a3329ffbdc5f0f1e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/fbf0e52a597b4c169ecd6ef1931ccb85, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/257cc7c7d4ff46d1b67d59edce8025bf, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/bee8b8cbe1f3491e89047ea67cf2f4ee, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/d7db5ce70800493989ecb047663e21d3] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp, totalSize=59.6 K 2024-12-04T14:59:08,119 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 735d255f8a594061a3329ffbdc5f0f1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733324343074 2024-12-04T14:59:08,120 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting fbf0e52a597b4c169ecd6ef1931ccb85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733324343624 2024-12-04T14:59:08,121 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 257cc7c7d4ff46d1b67d59edce8025bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733324343798 2024-12-04T14:59:08,124 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/d9469c995c084596948281fa62b27df9 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/d9469c995c084596948281fa62b27df9 2024-12-04T14:59:08,126 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting bee8b8cbe1f3491e89047ea67cf2f4ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733324344679 2024-12-04T14:59:08,129 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting d7db5ce70800493989ecb047663e21d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733324346303 2024-12-04T14:59:08,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:08,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47aad6a6d662b77a47b7fe8495dd267c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T14:59:08,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=A 2024-12-04T14:59:08,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:08,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=B 2024-12-04T14:59:08,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:08,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=C 2024-12-04T14:59:08,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:08,143 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 47aad6a6d662b77a47b7fe8495dd267c/A of 47aad6a6d662b77a47b7fe8495dd267c into d9469c995c084596948281fa62b27df9(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:08,143 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:08,143 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., storeName=47aad6a6d662b77a47b7fe8495dd267c/A, priority=11, startTime=1733324348027; duration=0sec 2024-12-04T14:59:08,143 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:08,143 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47aad6a6d662b77a47b7fe8495dd267c:A 2024-12-04T14:59:08,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T14:59:08,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:08,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:08,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:08,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:08,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:08,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T14:59:08,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:08,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412044f905bb204ca41fda7c4fc54213f4fd7_47aad6a6d662b77a47b7fe8495dd267c is 50, key is test_row_0/A:col10/1733324348129/Put/seqid=0 2024-12-04T14:59:08,160 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47aad6a6d662b77a47b7fe8495dd267c#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:08,160 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/fae8c59f7fd34df59250bbf981129fae is 50, key is test_row_0/C:col10/1733324346314/Put/seqid=0 2024-12-04T14:59:08,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741993_1169 (size=17534) 2024-12-04T14:59:08,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324408208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:08,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:08,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:08,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:08,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324348187, max timestamp=1733324348209Num uniques 859; 2024-12-04T14:59:08,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324348187, max timestamp=1733324348209Num uniques 859; 2024-12-04T14:59:08,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324348187, max timestamp=1733324348209Num uniques 859; 2024-12-04T14:59:08,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:59:08,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:59:08,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:59:08,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741994_1170 (size=12711) 2024-12-04T14:59:08,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T14:59:08,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:08,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:08,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:08,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:08,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:08,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:08,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324408302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:08,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:08,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324408320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:08,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324408410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:08,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T14:59:08,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T14:59:08,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:08,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:08,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:08,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:08,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:08,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:08,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:08,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324408530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:08,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:08,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:08,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:08,606 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:08,614 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412044f905bb204ca41fda7c4fc54213f4fd7_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412044f905bb204ca41fda7c4fc54213f4fd7_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:08,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T14:59:08,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:08,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:08,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:08,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:08,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:08,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:08,624 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/ff8757d3827943968a3f17688ce2bd11, store: [table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:08,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/ff8757d3827943968a3f17688ce2bd11 is 175, key is test_row_0/A:col10/1733324348129/Put/seqid=0 2024-12-04T14:59:08,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:08,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324408623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:08,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:08,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:08,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:08,636 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/fae8c59f7fd34df59250bbf981129fae as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/fae8c59f7fd34df59250bbf981129fae 2024-12-04T14:59:08,647 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 47aad6a6d662b77a47b7fe8495dd267c/C of 47aad6a6d662b77a47b7fe8495dd267c into fae8c59f7fd34df59250bbf981129fae(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:08,647 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:08,647 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., storeName=47aad6a6d662b77a47b7fe8495dd267c/C, priority=11, startTime=1733324348028; duration=0sec 2024-12-04T14:59:08,647 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:08,647 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47aad6a6d662b77a47b7fe8495dd267c:C 2024-12-04T14:59:08,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741995_1171 (size=48639) 2024-12-04T14:59:08,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T14:59:08,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:08,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:08,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:08,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:08,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:08,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:08,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324408840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:08,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:08,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:08,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:08,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T14:59:08,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:08,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:08,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:08,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:08,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:08,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324408930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:08,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:08,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:08,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T14:59:09,053 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/ff8757d3827943968a3f17688ce2bd11 2024-12-04T14:59:09,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/428196bb466e4eac97e17b4bd57fc1f7 is 50, key is test_row_0/B:col10/1733324348129/Put/seqid=0 2024-12-04T14:59:09,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741996_1172 (size=12301) 2024-12-04T14:59:09,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T14:59:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:09,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:09,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/428196bb466e4eac97e17b4bd57fc1f7 2024-12-04T14:59:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:09,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/2ca9d33ac4114d5da443f6fade5d7db7 is 50, key is test_row_0/C:col10/1733324348129/Put/seqid=0 2024-12-04T14:59:09,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741997_1173 (size=12301) 2024-12-04T14:59:09,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/2ca9d33ac4114d5da443f6fade5d7db7 2024-12-04T14:59:09,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/ff8757d3827943968a3f17688ce2bd11 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/ff8757d3827943968a3f17688ce2bd11 2024-12-04T14:59:09,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/ff8757d3827943968a3f17688ce2bd11, entries=250, sequenceid=309, filesize=47.5 K 2024-12-04T14:59:09,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/428196bb466e4eac97e17b4bd57fc1f7 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/428196bb466e4eac97e17b4bd57fc1f7 2024-12-04T14:59:09,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/428196bb466e4eac97e17b4bd57fc1f7, entries=150, sequenceid=309, filesize=12.0 K 2024-12-04T14:59:09,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/2ca9d33ac4114d5da443f6fade5d7db7 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2ca9d33ac4114d5da443f6fade5d7db7 2024-12-04T14:59:09,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2ca9d33ac4114d5da443f6fade5d7db7, entries=150, sequenceid=309, filesize=12.0 K 2024-12-04T14:59:09,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 47aad6a6d662b77a47b7fe8495dd267c in 1030ms, sequenceid=309, compaction requested=false 2024-12-04T14:59:09,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:09,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T14:59:09,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:09,232 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 47aad6a6d662b77a47b7fe8495dd267c 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-04T14:59:09,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=A 2024-12-04T14:59:09,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:09,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=B 2024-12-04T14:59:09,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:09,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=C 2024-12-04T14:59:09,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:09,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412046ec54a6e55244552b5a69944c78543f2_47aad6a6d662b77a47b7fe8495dd267c is 50, key is test_row_0/A:col10/1733324348280/Put/seqid=0 2024-12-04T14:59:09,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741998_1174 (size=12454) 2024-12-04T14:59:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:09,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:09,367 INFO [master/d214edae7bd2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T14:59:09,367 INFO [master/d214edae7bd2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T14:59:09,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324409383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:09,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324409440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:09,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324409490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:09,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:09,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:09,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:09,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:09,666 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412046ec54a6e55244552b5a69944c78543f2_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046ec54a6e55244552b5a69944c78543f2_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:09,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/d7588cd953c24db4a05d9c0dedee6c9a, store: [table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:09,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/d7588cd953c24db4a05d9c0dedee6c9a is 175, key is test_row_0/A:col10/1733324348280/Put/seqid=0 2024-12-04T14:59:09,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741999_1175 (size=31255) 2024-12-04T14:59:09,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324409701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:09,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:09,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:09,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:09,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T14:59:10,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324410010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:10,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:10,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:10,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:10,075 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=347, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/d7588cd953c24db4a05d9c0dedee6c9a 2024-12-04T14:59:10,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/8f10176d8b94472eb2464d62843eb266 is 50, key is test_row_0/B:col10/1733324348280/Put/seqid=0 2024-12-04T14:59:10,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742000_1176 (size=12301) 2024-12-04T14:59:10,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324410450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:10,453 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_1' in 47aad6a6d662b77a47b7fe8495dd267c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:10,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:10,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:10,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:10,502 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/8f10176d8b94472eb2464d62843eb266 2024-12-04T14:59:10,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/b871b286d4d046c2b4bc88dcde04886a is 50, key is test_row_0/C:col10/1733324348280/Put/seqid=0 2024-12-04T14:59:10,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742001_1177 (size=12301) 2024-12-04T14:59:10,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324410520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:10,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:10,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:10,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:10,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39330 deadline: 1733324410730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:10,732 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 47aad6a6d662b77a47b7fe8495dd267c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:10,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:10,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733324410740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:10,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:10,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39338 deadline: 1733324410740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:10,743 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_0' in 47aad6a6d662b77a47b7fe8495dd267c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:10,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:10,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:10,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:10,743 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 47aad6a6d662b77a47b7fe8495dd267c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:10,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:10,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:10,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:10,913 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/b871b286d4d046c2b4bc88dcde04886a 2024-12-04T14:59:10,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/d7588cd953c24db4a05d9c0dedee6c9a as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/d7588cd953c24db4a05d9c0dedee6c9a 2024-12-04T14:59:10,925 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/d7588cd953c24db4a05d9c0dedee6c9a, entries=150, sequenceid=347, filesize=30.5 K 2024-12-04T14:59:10,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/8f10176d8b94472eb2464d62843eb266 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/8f10176d8b94472eb2464d62843eb266 2024-12-04T14:59:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,932 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/8f10176d8b94472eb2464d62843eb266, entries=150, sequenceid=347, filesize=12.0 K 2024-12-04T14:59:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/b871b286d4d046c2b4bc88dcde04886a as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/b871b286d4d046c2b4bc88dcde04886a 2024-12-04T14:59:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,940 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/b871b286d4d046c2b4bc88dcde04886a, entries=150, sequenceid=347, filesize=12.0 K 2024-12-04T14:59:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,941 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 47aad6a6d662b77a47b7fe8495dd267c in 1710ms, sequenceid=347, compaction requested=true 2024-12-04T14:59:10,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:10,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:10,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-04T14:59:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-04T14:59:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,945 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-04T14:59:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,945 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1090 sec 2024-12-04T14:59:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 3.1140 sec 2024-12-04T14:59:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47aad6a6d662b77a47b7fe8495dd267c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T14:59:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=A 2024-12-04T14:59:11,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:11,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=B 2024-12-04T14:59:11,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:11,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=C 2024-12-04T14:59:11,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e22bb65cc71f4829b6663e898579b92a_47aad6a6d662b77a47b7fe8495dd267c is 50, key is test_row_0/A:col10/1733324351549/Put/seqid=0 2024-12-04T14:59:11,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742002_1178 (size=12454) 2024-12-04T14:59:11,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324411642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:11,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:11,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:11,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:11,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324411750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:11,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:11,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:11,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:11,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:11,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324411960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:11,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:11,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:11,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:11,974 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:11,980 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e22bb65cc71f4829b6663e898579b92a_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e22bb65cc71f4829b6663e898579b92a_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:11,981 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/f36db39a041848a19f81d651d4c93690, store: [table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:11,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/f36db39a041848a19f81d651d4c93690 is 175, key is test_row_0/A:col10/1733324351549/Put/seqid=0 2024-12-04T14:59:11,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742003_1179 (size=31251) 2024-12-04T14:59:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T14:59:11,990 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:11,991 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:59:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-04T14:59:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T14:59:11,993 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:59:11,994 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:59:11,994 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:59:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T14:59:12,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T14:59:12,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:12,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:12,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:12,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:12,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:12,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324412270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:12,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:12,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:12,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:12,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T14:59:12,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:12,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:12,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:12,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:12,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T14:59:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:12,391 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/f36db39a041848a19f81d651d4c93690 2024-12-04T14:59:12,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/75dfe4e1d28e4208a7b3b6f523679aa9 is 50, key is test_row_0/B:col10/1733324351549/Put/seqid=0 2024-12-04T14:59:12,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742004_1180 (size=9857) 2024-12-04T14:59:12,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T14:59:12,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:12,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:12,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:12,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:12,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:12,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324412460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:12,464 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_1' in 47aad6a6d662b77a47b7fe8495dd267c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:12,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:12,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:12,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:12,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T14:59:12,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:12,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:12,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:12,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:12,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T14:59:12,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T14:59:12,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:12,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:12,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:12,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:12,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:12,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324412780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:12,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/75dfe4e1d28e4208a7b3b6f523679aa9 2024-12-04T14:59:12,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/5e002d4423844596866b2610b51e3b7e is 50, key is test_row_0/C:col10/1733324351549/Put/seqid=0 2024-12-04T14:59:12,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742005_1181 (size=9857) 2024-12-04T14:59:12,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/5e002d4423844596866b2610b51e3b7e 2024-12-04T14:59:12,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/f36db39a041848a19f81d651d4c93690 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/f36db39a041848a19f81d651d4c93690 2024-12-04T14:59:12,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/f36db39a041848a19f81d651d4c93690, entries=150, sequenceid=359, filesize=30.5 K 2024-12-04T14:59:12,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/75dfe4e1d28e4208a7b3b6f523679aa9 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/75dfe4e1d28e4208a7b3b6f523679aa9 2024-12-04T14:59:12,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/75dfe4e1d28e4208a7b3b6f523679aa9, entries=100, sequenceid=359, filesize=9.6 K 2024-12-04T14:59:12,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/5e002d4423844596866b2610b51e3b7e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/5e002d4423844596866b2610b51e3b7e 2024-12-04T14:59:12,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/5e002d4423844596866b2610b51e3b7e, entries=100, sequenceid=359, filesize=9.6 K 2024-12-04T14:59:12,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 47aad6a6d662b77a47b7fe8495dd267c in 1288ms, sequenceid=359, compaction requested=true 2024-12-04T14:59:12,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:12,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47aad6a6d662b77a47b7fe8495dd267c:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:59:12,839 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:59:12,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:12,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47aad6a6d662b77a47b7fe8495dd267c:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:59:12,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:12,839 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:59:12,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47aad6a6d662b77a47b7fe8495dd267c:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:59:12,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:12,841 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142810 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:59:12,841 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47170 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:59:12,841 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 47aad6a6d662b77a47b7fe8495dd267c/A is initiating minor compaction (all files) 2024-12-04T14:59:12,841 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 47aad6a6d662b77a47b7fe8495dd267c/B is initiating minor compaction (all files) 2024-12-04T14:59:12,841 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47aad6a6d662b77a47b7fe8495dd267c/B in TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:12,841 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47aad6a6d662b77a47b7fe8495dd267c/A in TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:12,841 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/d9469c995c084596948281fa62b27df9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/ff8757d3827943968a3f17688ce2bd11, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/d7588cd953c24db4a05d9c0dedee6c9a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/f36db39a041848a19f81d651d4c93690] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp, totalSize=139.5 K 2024-12-04T14:59:12,841 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/446c89a74831498297bf7d8565114c94, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/428196bb466e4eac97e17b4bd57fc1f7, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/8f10176d8b94472eb2464d62843eb266, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/75dfe4e1d28e4208a7b3b6f523679aa9] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp, totalSize=46.1 K 2024-12-04T14:59:12,841 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:12,842 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/d9469c995c084596948281fa62b27df9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/ff8757d3827943968a3f17688ce2bd11, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/d7588cd953c24db4a05d9c0dedee6c9a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/f36db39a041848a19f81d651d4c93690] 2024-12-04T14:59:12,842 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 446c89a74831498297bf7d8565114c94, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733324346303 2024-12-04T14:59:12,842 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9469c995c084596948281fa62b27df9, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733324346303 2024-12-04T14:59:12,842 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff8757d3827943968a3f17688ce2bd11, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733324347487 2024-12-04T14:59:12,843 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 428196bb466e4eac97e17b4bd57fc1f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733324347487 2024-12-04T14:59:12,843 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f10176d8b94472eb2464d62843eb266, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733324348280 2024-12-04T14:59:12,843 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7588cd953c24db4a05d9c0dedee6c9a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733324348280 2024-12-04T14:59:12,844 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f36db39a041848a19f81d651d4c93690, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733324349357 2024-12-04T14:59:12,844 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 75dfe4e1d28e4208a7b3b6f523679aa9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733324349357 2024-12-04T14:59:12,852 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:12,855 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47aad6a6d662b77a47b7fe8495dd267c#B#compaction#151 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:12,856 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/fc027cfd580249dfa0943d23daadd0e9 is 50, key is test_row_0/B:col10/1733324351549/Put/seqid=0 2024-12-04T14:59:12,856 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204b566eaab37da49048559130d70fc3ca4_47aad6a6d662b77a47b7fe8495dd267c store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:12,858 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204b566eaab37da49048559130d70fc3ca4_47aad6a6d662b77a47b7fe8495dd267c, store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:12,858 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b566eaab37da49048559130d70fc3ca4_47aad6a6d662b77a47b7fe8495dd267c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:12,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742006_1182 (size=4469) 2024-12-04T14:59:12,870 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47aad6a6d662b77a47b7fe8495dd267c#A#compaction#150 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:12,871 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/2b88cc3f5dcc4a68937d8dff453b633f is 175, key is test_row_0/A:col10/1733324351549/Put/seqid=0 2024-12-04T14:59:12,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742007_1183 (size=12847) 2024-12-04T14:59:12,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742008_1184 (size=31908) 2024-12-04T14:59:12,891 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/2b88cc3f5dcc4a68937d8dff453b633f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/2b88cc3f5dcc4a68937d8dff453b633f 2024-12-04T14:59:12,897 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47aad6a6d662b77a47b7fe8495dd267c/A of 47aad6a6d662b77a47b7fe8495dd267c into 2b88cc3f5dcc4a68937d8dff453b633f(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:12,897 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:12,897 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., storeName=47aad6a6d662b77a47b7fe8495dd267c/A, priority=12, startTime=1733324352839; duration=0sec 2024-12-04T14:59:12,897 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:12,897 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47aad6a6d662b77a47b7fe8495dd267c:A 2024-12-04T14:59:12,897 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:59:12,900 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47170 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:59:12,900 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 47aad6a6d662b77a47b7fe8495dd267c/C is initiating minor compaction (all files) 2024-12-04T14:59:12,900 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47aad6a6d662b77a47b7fe8495dd267c/C in TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:12,900 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/fae8c59f7fd34df59250bbf981129fae, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2ca9d33ac4114d5da443f6fade5d7db7, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/b871b286d4d046c2b4bc88dcde04886a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/5e002d4423844596866b2610b51e3b7e] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp, totalSize=46.1 K 2024-12-04T14:59:12,900 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting fae8c59f7fd34df59250bbf981129fae, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733324346303 2024-12-04T14:59:12,901 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ca9d33ac4114d5da443f6fade5d7db7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733324347487 2024-12-04T14:59:12,901 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting b871b286d4d046c2b4bc88dcde04886a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733324348280 2024-12-04T14:59:12,901 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e002d4423844596866b2610b51e3b7e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733324349357 2024-12-04T14:59:12,916 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47aad6a6d662b77a47b7fe8495dd267c#C#compaction#152 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:12,917 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/919a3e7a180940ecbca41e15aa53004b is 50, key is test_row_0/C:col10/1733324351549/Put/seqid=0 2024-12-04T14:59:12,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T14:59:12,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:12,922 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 47aad6a6d662b77a47b7fe8495dd267c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T14:59:12,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=A 2024-12-04T14:59:12,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:12,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=B 2024-12-04T14:59:12,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:12,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=C 2024-12-04T14:59:12,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:12,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742009_1185 (size=12847) 2024-12-04T14:59:12,938 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/919a3e7a180940ecbca41e15aa53004b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/919a3e7a180940ecbca41e15aa53004b 2024-12-04T14:59:12,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412048326eb7559ff43d989880adc96473588_47aad6a6d662b77a47b7fe8495dd267c is 50, key is test_row_0/A:col10/1733324351638/Put/seqid=0 2024-12-04T14:59:12,948 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47aad6a6d662b77a47b7fe8495dd267c/C of 47aad6a6d662b77a47b7fe8495dd267c into 919a3e7a180940ecbca41e15aa53004b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:12,948 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:12,948 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., storeName=47aad6a6d662b77a47b7fe8495dd267c/C, priority=12, startTime=1733324352839; duration=0sec 2024-12-04T14:59:12,948 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:12,948 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47aad6a6d662b77a47b7fe8495dd267c:C 2024-12-04T14:59:12,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742010_1186 (size=12454) 2024-12-04T14:59:12,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:12,961 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412048326eb7559ff43d989880adc96473588_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048326eb7559ff43d989880adc96473588_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:12,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/76e0b95e0a7c40c78b32c1c95dea7a40, store: [table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:12,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/76e0b95e0a7c40c78b32c1c95dea7a40 is 175, key is test_row_0/A:col10/1733324351638/Put/seqid=0 2024-12-04T14:59:12,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742011_1187 (size=31255) 2024-12-04T14:59:13,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T14:59:13,289 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/fc027cfd580249dfa0943d23daadd0e9 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/fc027cfd580249dfa0943d23daadd0e9 2024-12-04T14:59:13,298 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47aad6a6d662b77a47b7fe8495dd267c/B of 47aad6a6d662b77a47b7fe8495dd267c into fc027cfd580249dfa0943d23daadd0e9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:13,298 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:13,298 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., storeName=47aad6a6d662b77a47b7fe8495dd267c/B, priority=12, startTime=1733324352839; duration=0sec 2024-12-04T14:59:13,299 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:13,299 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47aad6a6d662b77a47b7fe8495dd267c:B 2024-12-04T14:59:13,375 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/76e0b95e0a7c40c78b32c1c95dea7a40 2024-12-04T14:59:13,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/1faffb4e08094e7d970d919f705bf024 is 50, key is test_row_0/B:col10/1733324351638/Put/seqid=0 2024-12-04T14:59:13,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742012_1188 (size=12301) 2024-12-04T14:59:13,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:13,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:13,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:13,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324413811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:13,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:13,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:13,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:13,814 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/1faffb4e08094e7d970d919f705bf024 2024-12-04T14:59:13,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/11a1c63684a34abebbc7151dd70e9467 is 50, key is test_row_0/C:col10/1733324351638/Put/seqid=0 2024-12-04T14:59:13,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742013_1189 (size=12301) 2024-12-04T14:59:13,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324413920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:14,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324414130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:14,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:14,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:14,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T14:59:14,225 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/11a1c63684a34abebbc7151dd70e9467 2024-12-04T14:59:14,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/76e0b95e0a7c40c78b32c1c95dea7a40 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/76e0b95e0a7c40c78b32c1c95dea7a40 2024-12-04T14:59:14,235 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/76e0b95e0a7c40c78b32c1c95dea7a40, entries=150, sequenceid=380, filesize=30.5 K 2024-12-04T14:59:14,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/1faffb4e08094e7d970d919f705bf024 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/1faffb4e08094e7d970d919f705bf024 2024-12-04T14:59:14,241 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/1faffb4e08094e7d970d919f705bf024, entries=150, sequenceid=380, filesize=12.0 K 2024-12-04T14:59:14,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/11a1c63684a34abebbc7151dd70e9467 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/11a1c63684a34abebbc7151dd70e9467 2024-12-04T14:59:14,247 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/11a1c63684a34abebbc7151dd70e9467, entries=150, sequenceid=380, filesize=12.0 K 2024-12-04T14:59:14,248 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 47aad6a6d662b77a47b7fe8495dd267c in 1326ms, sequenceid=380, compaction requested=false 2024-12-04T14:59:14,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:14,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:14,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-04T14:59:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-04T14:59:14,251 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-04T14:59:14,251 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2560 sec 2024-12-04T14:59:14,253 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.2610 sec 2024-12-04T14:59:14,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:14,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47aad6a6d662b77a47b7fe8495dd267c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T14:59:14,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=A 2024-12-04T14:59:14,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:14,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=B 2024-12-04T14:59:14,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:14,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=C 2024-12-04T14:59:14,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:14,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c6b569d705254b2190af9fb072f6f9fc_47aad6a6d662b77a47b7fe8495dd267c is 50, key is test_row_0/A:col10/1733324353807/Put/seqid=0 2024-12-04T14:59:14,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742014_1190 (size=14994) 2024-12-04T14:59:14,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324414496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:14,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324414610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:14,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:14,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324414820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:14,857 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:14,865 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c6b569d705254b2190af9fb072f6f9fc_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c6b569d705254b2190af9fb072f6f9fc_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:14,867 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/ace0f9c49ce6421db84993688bb91411, store: [table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:14,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/ace0f9c49ce6421db84993688bb91411 is 175, key is test_row_0/A:col10/1733324353807/Put/seqid=0 2024-12-04T14:59:14,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742015_1191 (size=39949) 2024-12-04T14:59:14,883 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=395, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/ace0f9c49ce6421db84993688bb91411 2024-12-04T14:59:14,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/c1a4df41d0b94b228c168a1cc7f6362d is 50, key is test_row_0/B:col10/1733324353807/Put/seqid=0 2024-12-04T14:59:14,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742016_1192 (size=12301) 2024-12-04T14:59:14,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/c1a4df41d0b94b228c168a1cc7f6362d 2024-12-04T14:59:14,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/2b61183d3cff4548b91ec0ca555923b7 is 50, key is test_row_0/C:col10/1733324353807/Put/seqid=0 2024-12-04T14:59:14,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742017_1193 (size=12301) 2024-12-04T14:59:15,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324415130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:15,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/2b61183d3cff4548b91ec0ca555923b7 2024-12-04T14:59:15,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/ace0f9c49ce6421db84993688bb91411 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/ace0f9c49ce6421db84993688bb91411 2024-12-04T14:59:15,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/ace0f9c49ce6421db84993688bb91411, entries=200, sequenceid=395, filesize=39.0 K 2024-12-04T14:59:15,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/c1a4df41d0b94b228c168a1cc7f6362d as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/c1a4df41d0b94b228c168a1cc7f6362d 2024-12-04T14:59:15,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/c1a4df41d0b94b228c168a1cc7f6362d, entries=150, sequenceid=395, filesize=12.0 K 2024-12-04T14:59:15,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/2b61183d3cff4548b91ec0ca555923b7 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2b61183d3cff4548b91ec0ca555923b7 2024-12-04T14:59:15,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2b61183d3cff4548b91ec0ca555923b7, entries=150, sequenceid=395, filesize=12.0 K 2024-12-04T14:59:15,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 47aad6a6d662b77a47b7fe8495dd267c in 906ms, sequenceid=395, compaction requested=true 2024-12-04T14:59:15,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:15,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47aad6a6d662b77a47b7fe8495dd267c:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:59:15,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:15,349 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:15,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47aad6a6d662b77a47b7fe8495dd267c:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:59:15,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:15,349 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:15,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47aad6a6d662b77a47b7fe8495dd267c:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:59:15,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:15,351 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103112 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:15,351 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:15,351 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 47aad6a6d662b77a47b7fe8495dd267c/B is initiating minor compaction (all files) 2024-12-04T14:59:15,351 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 47aad6a6d662b77a47b7fe8495dd267c/A is initiating minor compaction (all files) 2024-12-04T14:59:15,351 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47aad6a6d662b77a47b7fe8495dd267c/A in TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:15,351 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47aad6a6d662b77a47b7fe8495dd267c/B in TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:15,351 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/fc027cfd580249dfa0943d23daadd0e9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/1faffb4e08094e7d970d919f705bf024, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/c1a4df41d0b94b228c168a1cc7f6362d] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp, totalSize=36.6 K 2024-12-04T14:59:15,351 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/2b88cc3f5dcc4a68937d8dff453b633f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/76e0b95e0a7c40c78b32c1c95dea7a40, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/ace0f9c49ce6421db84993688bb91411] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp, totalSize=100.7 K 2024-12-04T14:59:15,351 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:15,351 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/2b88cc3f5dcc4a68937d8dff453b633f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/76e0b95e0a7c40c78b32c1c95dea7a40, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/ace0f9c49ce6421db84993688bb91411] 2024-12-04T14:59:15,352 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting fc027cfd580249dfa0943d23daadd0e9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733324348287 2024-12-04T14:59:15,352 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1faffb4e08094e7d970d919f705bf024, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733324351627 2024-12-04T14:59:15,352 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b88cc3f5dcc4a68937d8dff453b633f, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733324348287 2024-12-04T14:59:15,352 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting c1a4df41d0b94b228c168a1cc7f6362d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733324353801 2024-12-04T14:59:15,352 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76e0b95e0a7c40c78b32c1c95dea7a40, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733324351627 2024-12-04T14:59:15,353 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting ace0f9c49ce6421db84993688bb91411, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733324353801 2024-12-04T14:59:15,366 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:15,380 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47aad6a6d662b77a47b7fe8495dd267c#B#compaction#160 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:15,381 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/b71c1d4756bc4fbd8f2dccf117074964 is 50, key is test_row_0/B:col10/1733324353807/Put/seqid=0 2024-12-04T14:59:15,395 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204c7b5778d07ef4c9e95d5f418777fa065_47aad6a6d662b77a47b7fe8495dd267c store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:15,397 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204c7b5778d07ef4c9e95d5f418777fa065_47aad6a6d662b77a47b7fe8495dd267c, store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:15,397 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c7b5778d07ef4c9e95d5f418777fa065_47aad6a6d662b77a47b7fe8495dd267c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:15,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742018_1194 (size=12949) 2024-12-04T14:59:15,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742019_1195 (size=4469) 2024-12-04T14:59:15,415 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47aad6a6d662b77a47b7fe8495dd267c#A#compaction#159 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:15,416 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/430694f4a842435baf9f399ba2986805 is 175, key is test_row_0/A:col10/1733324353807/Put/seqid=0 2024-12-04T14:59:15,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742020_1196 (size=31903) 2024-12-04T14:59:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:15,644 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47aad6a6d662b77a47b7fe8495dd267c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T14:59:15,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=A 2024-12-04T14:59:15,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:15,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=B 2024-12-04T14:59:15,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:15,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=C 2024-12-04T14:59:15,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:15,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040db90efbf7ce45fda83a930f60a3ceb5_47aad6a6d662b77a47b7fe8495dd267c is 50, key is test_row_0/A:col10/1733324354494/Put/seqid=0 2024-12-04T14:59:15,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742021_1197 (size=14994) 2024-12-04T14:59:15,664 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:15,667 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040db90efbf7ce45fda83a930f60a3ceb5_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040db90efbf7ce45fda83a930f60a3ceb5_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:15,668 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/fde6ad8fdd7340efa37d20dd1bf03109, store: [table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:15,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/fde6ad8fdd7340efa37d20dd1bf03109 is 175, key is test_row_0/A:col10/1733324354494/Put/seqid=0 2024-12-04T14:59:15,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742022_1198 (size=39949) 2024-12-04T14:59:15,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324415683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:15,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:15,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:15,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:15,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324415790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:15,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:15,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:15,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:15,807 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/b71c1d4756bc4fbd8f2dccf117074964 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/b71c1d4756bc4fbd8f2dccf117074964 2024-12-04T14:59:15,816 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47aad6a6d662b77a47b7fe8495dd267c/B of 47aad6a6d662b77a47b7fe8495dd267c into b71c1d4756bc4fbd8f2dccf117074964(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:15,816 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:15,816 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., storeName=47aad6a6d662b77a47b7fe8495dd267c/B, priority=13, startTime=1733324355349; duration=0sec 2024-12-04T14:59:15,816 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:15,816 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47aad6a6d662b77a47b7fe8495dd267c:B 2024-12-04T14:59:15,816 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:15,817 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:15,817 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 47aad6a6d662b77a47b7fe8495dd267c/C is initiating minor compaction (all files) 2024-12-04T14:59:15,818 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47aad6a6d662b77a47b7fe8495dd267c/C in TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:15,818 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/919a3e7a180940ecbca41e15aa53004b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/11a1c63684a34abebbc7151dd70e9467, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2b61183d3cff4548b91ec0ca555923b7] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp, totalSize=36.6 K 2024-12-04T14:59:15,818 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 919a3e7a180940ecbca41e15aa53004b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733324348287 2024-12-04T14:59:15,819 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 11a1c63684a34abebbc7151dd70e9467, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733324351627 2024-12-04T14:59:15,819 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b61183d3cff4548b91ec0ca555923b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733324353801 2024-12-04T14:59:15,830 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/430694f4a842435baf9f399ba2986805 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/430694f4a842435baf9f399ba2986805 2024-12-04T14:59:15,834 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47aad6a6d662b77a47b7fe8495dd267c#C#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:15,835 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/e2819870a3494adf95ef0031229f5c3f is 50, key is test_row_0/C:col10/1733324353807/Put/seqid=0 2024-12-04T14:59:15,836 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47aad6a6d662b77a47b7fe8495dd267c/A of 47aad6a6d662b77a47b7fe8495dd267c into 430694f4a842435baf9f399ba2986805(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:15,836 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:15,836 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., storeName=47aad6a6d662b77a47b7fe8495dd267c/A, priority=13, startTime=1733324355349; duration=0sec 2024-12-04T14:59:15,836 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:15,837 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47aad6a6d662b77a47b7fe8495dd267c:A 2024-12-04T14:59:15,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742023_1199 (size=12949) 2024-12-04T14:59:15,864 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/e2819870a3494adf95ef0031229f5c3f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/e2819870a3494adf95ef0031229f5c3f 2024-12-04T14:59:15,877 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47aad6a6d662b77a47b7fe8495dd267c/C of 47aad6a6d662b77a47b7fe8495dd267c into e2819870a3494adf95ef0031229f5c3f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:15,877 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:15,877 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., storeName=47aad6a6d662b77a47b7fe8495dd267c/C, priority=13, startTime=1733324355349; duration=0sec 2024-12-04T14:59:15,877 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:15,877 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47aad6a6d662b77a47b7fe8495dd267c:C 2024-12-04T14:59:16,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324416000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:16,075 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=415, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/fde6ad8fdd7340efa37d20dd1bf03109 2024-12-04T14:59:16,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/38c31d6fa66f4a18a46bbd233c127083 is 50, key is test_row_0/B:col10/1733324354494/Put/seqid=0 2024-12-04T14:59:16,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742024_1200 (size=12301) 2024-12-04T14:59:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T14:59:16,150 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:16,150 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:59:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-04T14:59:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T14:59:16,152 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:59:16,153 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:59:16,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:59:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T14:59:16,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T14:59:16,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:16,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:16,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:16,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:16,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:16,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324416310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:16,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T14:59:16,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:16,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:16,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:16,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:16,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T14:59:16,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39354 deadline: 1733324416480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:16,481 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_1' in 47aad6a6d662b77a47b7fe8495dd267c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:16,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/38c31d6fa66f4a18a46bbd233c127083 2024-12-04T14:59:16,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/2ae858b3abab468e9fa8996309df90b4 is 50, key is test_row_0/C:col10/1733324354494/Put/seqid=0 2024-12-04T14:59:16,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742025_1201 (size=12301) 2024-12-04T14:59:16,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T14:59:16,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:16,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:16,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:16,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:16,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:16,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T14:59:16,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:16,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:16,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:16,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:16,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:16,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T14:59:16,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324416820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:16,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T14:59:16,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:16,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:16,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:16,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:16,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:16,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/2ae858b3abab468e9fa8996309df90b4 2024-12-04T14:59:16,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/fde6ad8fdd7340efa37d20dd1bf03109 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/fde6ad8fdd7340efa37d20dd1bf03109 2024-12-04T14:59:16,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/fde6ad8fdd7340efa37d20dd1bf03109, entries=200, sequenceid=415, filesize=39.0 K 2024-12-04T14:59:16,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/38c31d6fa66f4a18a46bbd233c127083 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/38c31d6fa66f4a18a46bbd233c127083 2024-12-04T14:59:16,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/38c31d6fa66f4a18a46bbd233c127083, entries=150, sequenceid=415, filesize=12.0 K 2024-12-04T14:59:16,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/2ae858b3abab468e9fa8996309df90b4 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2ae858b3abab468e9fa8996309df90b4 2024-12-04T14:59:16,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2ae858b3abab468e9fa8996309df90b4, entries=150, sequenceid=415, filesize=12.0 K 2024-12-04T14:59:16,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 47aad6a6d662b77a47b7fe8495dd267c in 1339ms, sequenceid=415, compaction requested=false 2024-12-04T14:59:16,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:17,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T14:59:17,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:17,073 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 47aad6a6d662b77a47b7fe8495dd267c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T14:59:17,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=A 2024-12-04T14:59:17,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:17,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=B 2024-12-04T14:59:17,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:17,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=C 2024-12-04T14:59:17,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:17,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041a73387e85a8417ea47007afac78dcf2_47aad6a6d662b77a47b7fe8495dd267c is 50, key is test_row_0/A:col10/1733324355682/Put/seqid=0 2024-12-04T14:59:17,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742026_1202 (size=12454) 2024-12-04T14:59:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T14:59:17,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:17,494 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041a73387e85a8417ea47007afac78dcf2_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041a73387e85a8417ea47007afac78dcf2_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:17,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/0649ee88031748f385ba914e60e407de, store: [table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:17,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/0649ee88031748f385ba914e60e407de is 175, key is test_row_0/A:col10/1733324355682/Put/seqid=0 2024-12-04T14:59:17,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742027_1203 (size=31255) 2024-12-04T14:59:17,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:17,900 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=430, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/0649ee88031748f385ba914e60e407de 2024-12-04T14:59:17,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324417903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:17,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:17,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:17,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:17,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/f07bf3f57e75471c9da0b76c22235d27 is 50, key is test_row_0/B:col10/1733324355682/Put/seqid=0 2024-12-04T14:59:17,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742028_1204 (size=12301) 2024-12-04T14:59:18,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324418027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:18,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:18,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:18,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:18,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324418230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:18,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:18,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:18,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T14:59:18,349 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/f07bf3f57e75471c9da0b76c22235d27 2024-12-04T14:59:18,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/6c741c02a1494f468d2ab47ebe6ab615 is 50, key is test_row_0/C:col10/1733324355682/Put/seqid=0 2024-12-04T14:59:18,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742029_1205 (size=12301) 2024-12-04T14:59:18,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324418540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:18,760 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/6c741c02a1494f468d2ab47ebe6ab615 2024-12-04T14:59:18,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/0649ee88031748f385ba914e60e407de as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/0649ee88031748f385ba914e60e407de 2024-12-04T14:59:18,769 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/0649ee88031748f385ba914e60e407de, entries=150, sequenceid=430, filesize=30.5 K 2024-12-04T14:59:18,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/f07bf3f57e75471c9da0b76c22235d27 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/f07bf3f57e75471c9da0b76c22235d27 2024-12-04T14:59:18,774 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/f07bf3f57e75471c9da0b76c22235d27, entries=150, sequenceid=430, filesize=12.0 K 2024-12-04T14:59:18,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/6c741c02a1494f468d2ab47ebe6ab615 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/6c741c02a1494f468d2ab47ebe6ab615 2024-12-04T14:59:18,779 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/6c741c02a1494f468d2ab47ebe6ab615, entries=150, sequenceid=430, filesize=12.0 K 2024-12-04T14:59:18,780 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 47aad6a6d662b77a47b7fe8495dd267c in 1706ms, sequenceid=430, compaction requested=true 2024-12-04T14:59:18,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:18,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:18,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-04T14:59:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-04T14:59:18,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-04T14:59:18,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6280 sec 2024-12-04T14:59:18,783 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.6330 sec 2024-12-04T14:59:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:19,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324357883, max timestamp=1733324359051Num uniques 859; 2024-12-04T14:59:19,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47aad6a6d662b77a47b7fe8495dd267c 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-04T14:59:19,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324357883, max timestamp=1733324359051Num uniques 859; 2024-12-04T14:59:19,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324357883, max timestamp=1733324359051Num uniques 859; 2024-12-04T14:59:19,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:59:19,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:59:19,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:59:19,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=A 2024-12-04T14:59:19,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:19,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=B 2024-12-04T14:59:19,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:19,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=C 2024-12-04T14:59:19,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:19,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f2436ea3141c4214a7111264844f729c_47aad6a6d662b77a47b7fe8495dd267c is 50, key is test_row_0/A:col10/1733324357888/Put/seqid=0 2024-12-04T14:59:19,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742030_1206 (size=14994) 2024-12-04T14:59:19,065 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:19,068 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f2436ea3141c4214a7111264844f729c_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f2436ea3141c4214a7111264844f729c_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:19,082 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/cbd82a5f708446d4a70d1ae518c2531f, store: [table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:19,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/cbd82a5f708446d4a70d1ae518c2531f is 175, key is test_row_0/A:col10/1733324357888/Put/seqid=0 2024-12-04T14:59:19,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742031_1207 (size=39949) 2024-12-04T14:59:19,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324359121, max timestamp=1733324359136Num uniques 859; 2024-12-04T14:59:19,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324359121, max timestamp=1733324359136Num uniques 859; 2024-12-04T14:59:19,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324359121, max timestamp=1733324359136Num uniques 859; 2024-12-04T14:59:19,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:19,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:19,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:19,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733324359166, max timestamp=1733324359208Num uniques 1068; 2024-12-04T14:59:19,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733324359166, max timestamp=1733324359208Num uniques 1068; 2024-12-04T14:59:19,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733324359166, max timestamp=1733324359208Num uniques 1068; 2024-12-04T14:59:19,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-04T14:59:19,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-04T14:59:19,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-04T14:59:19,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733324359246, max timestamp=1733324359278Num uniques 1077; 2024-12-04T14:59:19,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-04T14:59:19,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733324359246, max timestamp=1733324359278Num uniques 1077; 2024-12-04T14:59:19,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-04T14:59:19,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733324359246, max timestamp=1733324359278Num uniques 1077; 2024-12-04T14:59:19,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-04T14:59:19,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733324359321, max timestamp=1733324359342Num uniques 1136; 2024-12-04T14:59:19,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-04T14:59:19,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733324359321, max timestamp=1733324359342Num uniques 1136; 2024-12-04T14:59:19,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-04T14:59:19,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733324359321, max timestamp=1733324359342Num uniques 1136; 2024-12-04T14:59:19,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-04T14:59:19,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733324359383, max timestamp=1733324359426Num uniques 1045; 2024-12-04T14:59:19,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-04T14:59:19,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733324359383, max timestamp=1733324359426Num uniques 1045; 2024-12-04T14:59:19,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733324359383, max timestamp=1733324359426Num uniques 1045; 2024-12-04T14:59:19,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-04T14:59:19,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-04T14:59:19,487 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=452, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/cbd82a5f708446d4a70d1ae518c2531f 2024-12-04T14:59:19,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/e43ddca178094e00b24442df7020d49a is 50, key is test_row_0/B:col10/1733324357888/Put/seqid=0 2024-12-04T14:59:19,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742032_1208 (size=12301) 2024-12-04T14:59:19,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733324359477, max timestamp=1733324359506Num uniques 1104; 2024-12-04T14:59:19,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-12-04T14:59:19,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733324359477, max timestamp=1733324359506Num uniques 1104; 2024-12-04T14:59:19,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-12-04T14:59:19,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733324359477, max timestamp=1733324359506Num uniques 1104; 2024-12-04T14:59:19,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-12-04T14:59:19,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733324359519, max timestamp=1733324359566Num uniques 1063; 2024-12-04T14:59:19,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=12038, suffix heap size=138520,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=263 2024-12-04T14:59:19,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733324359519, max timestamp=1733324359566Num uniques 1063; 2024-12-04T14:59:19,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=12038, suffix heap size=138520,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=263 2024-12-04T14:59:19,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733324359519, max timestamp=1733324359566Num uniques 1063; 2024-12-04T14:59:19,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=12038, suffix heap size=138520,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=263 2024-12-04T14:59:19,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1733324359593, max timestamp=1733324359622Num uniques 1122; 2024-12-04T14:59:19,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1733324359593, max timestamp=1733324359622Num uniques 1122; 2024-12-04T14:59:19,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=12452, suffix heap size=144656,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=272 2024-12-04T14:59:19,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=12452, suffix heap size=144656,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=272 2024-12-04T14:59:19,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1733324359593, max timestamp=1733324359622Num uniques 1122; 2024-12-04T14:59:19,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=12452, suffix heap size=144656,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=272 2024-12-04T14:59:19,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733324359668, max timestamp=1733324359692Num uniques 1131; 2024-12-04T14:59:19,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-12-04T14:59:19,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733324359668, max timestamp=1733324359692Num uniques 1131; 2024-12-04T14:59:19,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-12-04T14:59:19,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733324359668, max timestamp=1733324359692Num uniques 1131; 2024-12-04T14:59:19,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-12-04T14:59:19,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733324359724, max timestamp=1733324359753Num uniques 1090; 2024-12-04T14:59:19,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-12-04T14:59:19,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733324359724, max timestamp=1733324359753Num uniques 1090; 2024-12-04T14:59:19,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-12-04T14:59:19,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733324359724, max timestamp=1733324359753Num uniques 1090; 2024-12-04T14:59:19,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-12-04T14:59:19,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733324359794, max timestamp=1733324359822Num uniques 1099; 2024-12-04T14:59:19,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-12-04T14:59:19,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733324359794, max timestamp=1733324359822Num uniques 1099; 2024-12-04T14:59:19,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-12-04T14:59:19,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733324359794, max timestamp=1733324359822Num uniques 1099; 2024-12-04T14:59:19,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-12-04T14:59:19,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733324359849, max timestamp=1733324359894Num uniques 1108; 2024-12-04T14:59:19,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733324359849, max timestamp=1733324359894Num uniques 1108; 2024-12-04T14:59:19,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-12-04T14:59:19,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733324359849, max timestamp=1733324359894Num uniques 1108; 2024-12-04T14:59:19,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-12-04T14:59:19,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-12-04T14:59:19,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/e43ddca178094e00b24442df7020d49a 2024-12-04T14:59:19,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/0a8b3aa64f61495897e8d86f74435a59 is 50, key is test_row_0/C:col10/1733324357888/Put/seqid=0 2024-12-04T14:59:19,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742033_1209 (size=12301) 2024-12-04T14:59:19,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733324359926, max timestamp=1733324359967Num uniques 1067; 2024-12-04T14:59:19,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-12-04T14:59:19,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733324359926, max timestamp=1733324359967Num uniques 1067; 2024-12-04T14:59:19,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-12-04T14:59:19,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733324359926, max timestamp=1733324359967Num uniques 1067; 2024-12-04T14:59:19,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-12-04T14:59:20,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733324359989, max timestamp=1733324360043Num uniques 1076; 2024-12-04T14:59:20,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733324359989, max timestamp=1733324360043Num uniques 1076; 2024-12-04T14:59:20,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733324359989, max timestamp=1733324360043Num uniques 1076; 2024-12-04T14:59:20,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=8056, suffix heap size=139872,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-12-04T14:59:20,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=8056, suffix heap size=139872,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-12-04T14:59:20,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=8056, suffix heap size=139872,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-12-04T14:59:20,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1733324360088, max timestamp=1733324360123Num uniques 1035; 2024-12-04T14:59:20,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=13050, suffix heap size=135608,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=285 2024-12-04T14:59:20,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1733324360088, max timestamp=1733324360123Num uniques 1035; 2024-12-04T14:59:20,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=13050, suffix heap size=135608,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=285 2024-12-04T14:59:20,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1733324360088, max timestamp=1733324360123Num uniques 1035; 2024-12-04T14:59:20,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=13050, suffix heap size=135608,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=285 2024-12-04T14:59:20,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1733324360170, max timestamp=1733324360192Num uniques 1144; 2024-12-04T14:59:20,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=11174, suffix heap size=146944,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=244 2024-12-04T14:59:20,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1733324360170, max timestamp=1733324360192Num uniques 1144; 2024-12-04T14:59:20,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=11174, suffix heap size=146944,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=244 2024-12-04T14:59:20,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1733324360170, max timestamp=1733324360192Num uniques 1144; 2024-12-04T14:59:20,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=11174, suffix heap size=146944,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=244 2024-12-04T14:59:20,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=253, cellSize=11585, totalHeapSize=26552, min timestamp=1733324360244, max timestamp=1733324360262Num uniques 1103; 2024-12-04T14:59:20,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=11585, suffix heap size=142680,new segment heap size=26552 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=253 2024-12-04T14:59:20,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=253, cellSize=11585, totalHeapSize=26552, min timestamp=1733324360244, max timestamp=1733324360262Num uniques 1103; 2024-12-04T14:59:20,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=253, cellSize=11585, totalHeapSize=26552, min timestamp=1733324360244, max timestamp=1733324360262Num uniques 1103; 2024-12-04T14:59:20,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=11585, suffix heap size=142680,new segment heap size=26552 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=253 2024-12-04T14:59:20,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=11585, suffix heap size=142680,new segment heap size=26552 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=253 2024-12-04T14:59:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T14:59:20,320 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:20,320 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:59:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-04T14:59:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T14:59:20,322 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:59:20,322 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:59:20,322 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:59:20,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/0a8b3aa64f61495897e8d86f74435a59 2024-12-04T14:59:20,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1733324360309, max timestamp=1733324360331Num uniques 1112; 2024-12-04T14:59:20,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50922, new segment data size=9702, suffix heap size=143616,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1112, new segment cells count=212 2024-12-04T14:59:20,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1733324360309, max timestamp=1733324360331Num uniques 1112; 2024-12-04T14:59:20,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50922, new segment data size=9702, suffix heap size=143616,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1112, new segment cells count=212 2024-12-04T14:59:20,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1733324360309, max timestamp=1733324360331Num uniques 1112; 2024-12-04T14:59:20,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50922, new segment data size=9702, suffix heap size=143616,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1112, new segment cells count=212 2024-12-04T14:59:20,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/cbd82a5f708446d4a70d1ae518c2531f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/cbd82a5f708446d4a70d1ae518c2531f 2024-12-04T14:59:20,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/cbd82a5f708446d4a70d1ae518c2531f, entries=200, sequenceid=452, filesize=39.0 K 2024-12-04T14:59:20,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/e43ddca178094e00b24442df7020d49a as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/e43ddca178094e00b24442df7020d49a 2024-12-04T14:59:20,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/e43ddca178094e00b24442df7020d49a, entries=150, sequenceid=452, filesize=12.0 K 2024-12-04T14:59:20,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/0a8b3aa64f61495897e8d86f74435a59 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/0a8b3aa64f61495897e8d86f74435a59 2024-12-04T14:59:20,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/0a8b3aa64f61495897e8d86f74435a59, entries=150, sequenceid=452, filesize=12.0 K 2024-12-04T14:59:20,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=67.09 KB/68700 for 47aad6a6d662b77a47b7fe8495dd267c in 1305ms, sequenceid=452, compaction requested=true 2024-12-04T14:59:20,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:20,359 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:59:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:20,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47aad6a6d662b77a47b7fe8495dd267c:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:59:20,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:20,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47aad6a6d662b77a47b7fe8495dd267c:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:59:20,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:20,360 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:59:20,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47aad6a6d662b77a47b7fe8495dd267c:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:59:20,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:20,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47aad6a6d662b77a47b7fe8495dd267c 3/3 column families, dataSize=73.80 KB heapSize=200.72 KB 2024-12-04T14:59:20,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=A 2024-12-04T14:59:20,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:20,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=B 2024-12-04T14:59:20,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:20,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=C 2024-12-04T14:59:20,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:20,361 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:59:20,362 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 47aad6a6d662b77a47b7fe8495dd267c/A is initiating minor compaction (all files) 2024-12-04T14:59:20,362 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47aad6a6d662b77a47b7fe8495dd267c/A in TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:20,362 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/430694f4a842435baf9f399ba2986805, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/fde6ad8fdd7340efa37d20dd1bf03109, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/0649ee88031748f385ba914e60e407de, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/cbd82a5f708446d4a70d1ae518c2531f] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp, totalSize=139.7 K 2024-12-04T14:59:20,362 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:20,362 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/430694f4a842435baf9f399ba2986805, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/fde6ad8fdd7340efa37d20dd1bf03109, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/0649ee88031748f385ba914e60e407de, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/cbd82a5f708446d4a70d1ae518c2531f] 2024-12-04T14:59:20,362 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:59:20,362 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 47aad6a6d662b77a47b7fe8495dd267c/B is initiating minor compaction (all files) 2024-12-04T14:59:20,362 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47aad6a6d662b77a47b7fe8495dd267c/B in TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:20,362 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/b71c1d4756bc4fbd8f2dccf117074964, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/38c31d6fa66f4a18a46bbd233c127083, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/f07bf3f57e75471c9da0b76c22235d27, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/e43ddca178094e00b24442df7020d49a] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp, totalSize=48.7 K 2024-12-04T14:59:20,363 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 430694f4a842435baf9f399ba2986805, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733324353801 2024-12-04T14:59:20,363 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting b71c1d4756bc4fbd8f2dccf117074964, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733324353801 2024-12-04T14:59:20,363 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting fde6ad8fdd7340efa37d20dd1bf03109, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733324354478 2024-12-04T14:59:20,363 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 38c31d6fa66f4a18a46bbd233c127083, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733324354478 2024-12-04T14:59:20,363 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0649ee88031748f385ba914e60e407de, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1733324355656 2024-12-04T14:59:20,364 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting f07bf3f57e75471c9da0b76c22235d27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1733324355656 2024-12-04T14:59:20,364 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbd82a5f708446d4a70d1ae518c2531f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1733324357888 2024-12-04T14:59:20,364 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting e43ddca178094e00b24442df7020d49a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1733324357888 2024-12-04T14:59:20,372 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:20,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120456724c33c0c2497a867f0b2a0e212e7b_47aad6a6d662b77a47b7fe8495dd267c is 50, key is test_row_0/A:col10/1733324360358/Put/seqid=0 2024-12-04T14:59:20,374 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204c142b309c1eb444ab5bb061ce54e37f0_47aad6a6d662b77a47b7fe8495dd267c store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:20,376 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204c142b309c1eb444ab5bb061ce54e37f0_47aad6a6d662b77a47b7fe8495dd267c, store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:20,376 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47aad6a6d662b77a47b7fe8495dd267c#B#compaction#173 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:20,376 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c142b309c1eb444ab5bb061ce54e37f0_47aad6a6d662b77a47b7fe8495dd267c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:20,376 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/caabf27702c0435c802309cb75ecc31a is 50, key is test_row_0/B:col10/1733324357888/Put/seqid=0 2024-12-04T14:59:20,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742034_1210 (size=17534) 2024-12-04T14:59:20,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742035_1211 (size=4469) 2024-12-04T14:59:20,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742036_1212 (size=13085) 2024-12-04T14:59:20,396 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/caabf27702c0435c802309cb75ecc31a as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/caabf27702c0435c802309cb75ecc31a 2024-12-04T14:59:20,401 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47aad6a6d662b77a47b7fe8495dd267c/B of 47aad6a6d662b77a47b7fe8495dd267c into caabf27702c0435c802309cb75ecc31a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:20,401 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:20,401 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., storeName=47aad6a6d662b77a47b7fe8495dd267c/B, priority=12, startTime=1733324360359; duration=0sec 2024-12-04T14:59:20,401 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:20,401 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47aad6a6d662b77a47b7fe8495dd267c:B 2024-12-04T14:59:20,402 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:59:20,403 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:59:20,403 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 47aad6a6d662b77a47b7fe8495dd267c/C is initiating minor compaction (all files) 2024-12-04T14:59:20,403 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47aad6a6d662b77a47b7fe8495dd267c/C in TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:20,403 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/e2819870a3494adf95ef0031229f5c3f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2ae858b3abab468e9fa8996309df90b4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/6c741c02a1494f468d2ab47ebe6ab615, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/0a8b3aa64f61495897e8d86f74435a59] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp, totalSize=48.7 K 2024-12-04T14:59:20,403 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting e2819870a3494adf95ef0031229f5c3f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733324353801 2024-12-04T14:59:20,403 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ae858b3abab468e9fa8996309df90b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733324354478 2024-12-04T14:59:20,404 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c741c02a1494f468d2ab47ebe6ab615, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1733324355656 2024-12-04T14:59:20,404 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a8b3aa64f61495897e8d86f74435a59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1733324357888 2024-12-04T14:59:20,411 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47aad6a6d662b77a47b7fe8495dd267c#C#compaction#174 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:20,412 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/2e0125766821421c93f5bbe5c41d3eb6 is 50, key is test_row_0/C:col10/1733324357888/Put/seqid=0 2024-12-04T14:59:20,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742037_1213 (size=13085) 2024-12-04T14:59:20,425 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/2e0125766821421c93f5bbe5c41d3eb6 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2e0125766821421c93f5bbe5c41d3eb6 2024-12-04T14:59:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T14:59:20,431 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47aad6a6d662b77a47b7fe8495dd267c/C of 47aad6a6d662b77a47b7fe8495dd267c into 2e0125766821421c93f5bbe5c41d3eb6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:20,431 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:20,431 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., storeName=47aad6a6d662b77a47b7fe8495dd267c/C, priority=12, startTime=1733324360360; duration=0sec 2024-12-04T14:59:20,431 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:20,431 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47aad6a6d662b77a47b7fe8495dd267c:C 2024-12-04T14:59:20,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 544 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324420442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:20,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T14:59:20,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:20,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:20,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:20,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:20,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:20,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:20,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 545 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324420550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:20,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:20,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:20,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:20,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T14:59:20,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:20,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:20,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:20,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:20,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:20,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:20,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T14:59:20,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:20,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39330 deadline: 1733324420750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:20,752 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 47aad6a6d662b77a47b7fe8495dd267c of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:20,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:20,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:20,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:20,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:20,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 546 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324420760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:20,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:20,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:20,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:20,770 INFO [Thread-613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-613. 2024-12-04T14:59:20,770 DEBUG [Thread-613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:20,770 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:20,770 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:20,770 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:20,772 INFO [Thread-615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-615. 2024-12-04T14:59:20,772 DEBUG [Thread-615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:20,772 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:20,772 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:20,772 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:20,772 INFO [Thread-617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-617. 2024-12-04T14:59:20,772 DEBUG [Thread-617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:20,772 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:20,772 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:20,772 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:20,773 INFO [Thread-611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-611. 2024-12-04T14:59:20,773 DEBUG [Thread-611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:20,773 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:20,773 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:20,773 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:20,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T14:59:20,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:20,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:20,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:20,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:20,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:20,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:20,786 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:20,789 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120456724c33c0c2497a867f0b2a0e212e7b_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120456724c33c0c2497a867f0b2a0e212e7b_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:20,790 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47aad6a6d662b77a47b7fe8495dd267c#A#compaction#172 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:20,790 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/fc51889d71404473bf6873137b6bd13b, store: [table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:20,791 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/6a785c0e7a5a4c8a9069ca77f79aae24 is 175, key is test_row_0/A:col10/1733324357888/Put/seqid=0 2024-12-04T14:59:20,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/fc51889d71404473bf6873137b6bd13b is 175, key is test_row_0/A:col10/1733324360358/Put/seqid=0 2024-12-04T14:59:20,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742038_1214 (size=32039) 2024-12-04T14:59:20,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742039_1215 (size=48639) 2024-12-04T14:59:20,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:20,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39338 deadline: 1733324420810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:20,811 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 47aad6a6d662b77a47b7fe8495dd267c of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18301 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:20,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:20,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:20,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:20,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:20,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733324420831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:20,831 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_0' in 47aad6a6d662b77a47b7fe8495dd267c of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18328 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:20,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T14:59:20,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:20,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:20,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:20,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:20,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T14:59:21,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 547 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324421071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:21,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:21,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:21,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:21,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T14:59:21,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:21,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:21,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:21,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:21,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:21,196 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=771, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/fc51889d71404473bf6873137b6bd13b 2024-12-04T14:59:21,207 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/6a785c0e7a5a4c8a9069ca77f79aae24 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6a785c0e7a5a4c8a9069ca77f79aae24 2024-12-04T14:59:21,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/4d6cc26be55e4837a45015311302ccd9 is 50, key is test_row_0/B:col10/1733324360358/Put/seqid=0 2024-12-04T14:59:21,213 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47aad6a6d662b77a47b7fe8495dd267c/A of 47aad6a6d662b77a47b7fe8495dd267c into 6a785c0e7a5a4c8a9069ca77f79aae24(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:21,213 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:21,213 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., storeName=47aad6a6d662b77a47b7fe8495dd267c/A, priority=12, startTime=1733324360359; duration=0sec 2024-12-04T14:59:21,213 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:21,213 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47aad6a6d662b77a47b7fe8495dd267c:A 2024-12-04T14:59:21,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742040_1216 (size=12301) 2024-12-04T14:59:21,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T14:59:21,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:21,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:21,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:21,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:21,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:21,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:21,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T14:59:21,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:21,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:21,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:21,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:21,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T14:59:21,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T14:59:21,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:21,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:21,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:21,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:21,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:21,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:21,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 548 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1733324421581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:21,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:21,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:21,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:59:21,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=771 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/4d6cc26be55e4837a45015311302ccd9 2024-12-04T14:59:21,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/77c05b533c2448d482aa8e022bea95ce is 50, key is test_row_0/C:col10/1733324360358/Put/seqid=0 2024-12-04T14:59:21,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742041_1217 (size=12301) 2024-12-04T14:59:21,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T14:59:21,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:21,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:21,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:21,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:21,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:21,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T14:59:21,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:21,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:21,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:21,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:21,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:22,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T14:59:22,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:22,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:22,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:22,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:22,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:22,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=771 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/77c05b533c2448d482aa8e022bea95ce 2024-12-04T14:59:22,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/fc51889d71404473bf6873137b6bd13b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/fc51889d71404473bf6873137b6bd13b 2024-12-04T14:59:22,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/fc51889d71404473bf6873137b6bd13b, entries=250, sequenceid=771, filesize=47.5 K 2024-12-04T14:59:22,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/4d6cc26be55e4837a45015311302ccd9 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/4d6cc26be55e4837a45015311302ccd9 2024-12-04T14:59:22,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/4d6cc26be55e4837a45015311302ccd9, entries=150, sequenceid=771, filesize=12.0 K 2024-12-04T14:59:22,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/77c05b533c2448d482aa8e022bea95ce as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/77c05b533c2448d482aa8e022bea95ce 2024-12-04T14:59:22,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/77c05b533c2448d482aa8e022bea95ce, entries=150, sequenceid=771, filesize=12.0 K 2024-12-04T14:59:22,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~200.67 KB/205488, currentSize=107.34 KB/109920 for 47aad6a6d662b77a47b7fe8495dd267c in 1697ms, sequenceid=771, compaction requested=false 2024-12-04T14:59:22,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:22,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T14:59:22,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:22,175 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 47aad6a6d662b77a47b7fe8495dd267c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-04T14:59:22,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=A 2024-12-04T14:59:22,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:22,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=B 2024-12-04T14:59:22,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:22,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=C 2024-12-04T14:59:22,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:22,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204ed6b6bd95f2744ebbf1ece9ab534747d_47aad6a6d662b77a47b7fe8495dd267c is 50, key is test_row_0/A:col10/1733324360429/Put/seqid=0 2024-12-04T14:59:22,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742042_1218 (size=12454) 2024-12-04T14:59:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T14:59:22,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:22,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. as already flushing 2024-12-04T14:59:22,593 INFO [Thread-604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-604. 2024-12-04T14:59:22,593 DEBUG [Thread-604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:22,593 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:22,593 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:22,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:22,597 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204ed6b6bd95f2744ebbf1ece9ab534747d_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ed6b6bd95f2744ebbf1ece9ab534747d_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:22,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/36aa448620504b4c867900c1e4a8b7ab, store: [table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:22,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/36aa448620504b4c867900c1e4a8b7ab is 175, key is test_row_0/A:col10/1733324360429/Put/seqid=0 2024-12-04T14:59:22,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742043_1219 (size=31255) 2024-12-04T14:59:23,004 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=793, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/36aa448620504b4c867900c1e4a8b7ab 2024-12-04T14:59:23,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/38d21b2481a9466692f1595935b55de6 is 50, key is test_row_0/B:col10/1733324360429/Put/seqid=0 2024-12-04T14:59:23,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742044_1220 (size=12301) 2024-12-04T14:59:23,226 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T14:59:23,416 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=793 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/38d21b2481a9466692f1595935b55de6 2024-12-04T14:59:23,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/a0223d0bdc9248818884392da0d5eb82 is 50, key is test_row_0/C:col10/1733324360429/Put/seqid=0 2024-12-04T14:59:23,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742045_1221 (size=12301) 2024-12-04T14:59:23,828 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=793 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/a0223d0bdc9248818884392da0d5eb82 2024-12-04T14:59:23,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/36aa448620504b4c867900c1e4a8b7ab as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/36aa448620504b4c867900c1e4a8b7ab 2024-12-04T14:59:23,840 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/36aa448620504b4c867900c1e4a8b7ab, entries=150, sequenceid=793, filesize=30.5 K 2024-12-04T14:59:23,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/38d21b2481a9466692f1595935b55de6 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/38d21b2481a9466692f1595935b55de6 2024-12-04T14:59:23,848 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/38d21b2481a9466692f1595935b55de6, entries=150, sequenceid=793, filesize=12.0 K 2024-12-04T14:59:23,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/a0223d0bdc9248818884392da0d5eb82 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/a0223d0bdc9248818884392da0d5eb82 2024-12-04T14:59:23,854 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/a0223d0bdc9248818884392da0d5eb82, entries=150, sequenceid=793, filesize=12.0 K 2024-12-04T14:59:23,855 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=6.71 KB/6870 for 47aad6a6d662b77a47b7fe8495dd267c in 1679ms, sequenceid=793, compaction requested=true 2024-12-04T14:59:23,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 47aad6a6d662b77a47b7fe8495dd267c: 2024-12-04T14:59:23,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:23,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-04T14:59:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-04T14:59:23,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-04T14:59:23,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5340 sec 2024-12-04T14:59:23,859 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 3.5380 sec 2024-12-04T14:59:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T14:59:24,482 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:26,583 INFO [Thread-606 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-606. 2024-12-04T14:59:26,583 DEBUG [Thread-606 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:26,583 DEBUG [Thread-606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:26,583 DEBUG [Thread-606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:26,584 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:30,814 INFO [Thread-602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-602. 2024-12-04T14:59:30,814 DEBUG [Thread-602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:30,815 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:30,815 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:30,815 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:30,856 INFO [Thread-608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-608. 2024-12-04T14:59:30,857 DEBUG [Thread-608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:30,857 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:30,857 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:30,857 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:30,934 INFO [Thread-600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-600. 2024-12-04T14:59:30,934 DEBUG [Thread-600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:30,934 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:30,935 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:30,935 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:30,935 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T14:59:30,936 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-12-04T14:59:30,936 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-12-04T14:59:30,936 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 492 2024-12-04T14:59:30,936 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 163 2024-12-04T14:59:30,936 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-12-04T14:59:30,936 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T14:59:30,936 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3440 2024-12-04T14:59:30,936 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3433 2024-12-04T14:59:30,936 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T14:59:30,936 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1521 2024-12-04T14:59:30,936 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4562 rows 2024-12-04T14:59:30,936 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1521 2024-12-04T14:59:30,936 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4561 rows 2024-12-04T14:59:30,936 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T14:59:30,936 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T14:59:30,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:30,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:30,937 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:30,938 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T14:59:30,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T14:59:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T14:59:30,944 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324370943"}]},"ts":"1733324370943"} 2024-12-04T14:59:30,946 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T14:59:30,946 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T14:59:30,946 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T14:59:30,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47aad6a6d662b77a47b7fe8495dd267c, UNASSIGN}] 2024-12-04T14:59:30,948 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47aad6a6d662b77a47b7fe8495dd267c, UNASSIGN 2024-12-04T14:59:30,949 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=47aad6a6d662b77a47b7fe8495dd267c, regionState=CLOSING, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:30,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47aad6a6d662b77a47b7fe8495dd267c, UNASSIGN because future has completed 2024-12-04T14:59:30,950 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T14:59:30,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801}] 2024-12-04T14:59:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T14:59:31,103 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(122): Close 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:31,104 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T14:59:31,104 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1722): Closing 47aad6a6d662b77a47b7fe8495dd267c, disabling compactions & flushes 2024-12-04T14:59:31,104 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:31,104 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:31,104 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. after waiting 0 ms 2024-12-04T14:59:31,104 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:31,105 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(2902): Flushing 47aad6a6d662b77a47b7fe8495dd267c 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-04T14:59:31,106 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=A 2024-12-04T14:59:31,106 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:31,106 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=B 2024-12-04T14:59:31,106 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:31,106 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47aad6a6d662b77a47b7fe8495dd267c, store=C 2024-12-04T14:59:31,106 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:31,116 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040c12183f57f1415ba0a4e3e5220995eb_47aad6a6d662b77a47b7fe8495dd267c is 50, key is test_row_0/A:col10/1733324370931/Put/seqid=0 2024-12-04T14:59:31,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742046_1222 (size=12454) 2024-12-04T14:59:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T14:59:31,521 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:31,527 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040c12183f57f1415ba0a4e3e5220995eb_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040c12183f57f1415ba0a4e3e5220995eb_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:31,528 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/bf76421f46b04dc3ae7d49d7bae0ff18, store: [table=TestAcidGuarantees family=A region=47aad6a6d662b77a47b7fe8495dd267c] 2024-12-04T14:59:31,529 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/bf76421f46b04dc3ae7d49d7bae0ff18 is 175, key is test_row_0/A:col10/1733324370931/Put/seqid=0 2024-12-04T14:59:31,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742047_1223 (size=31255) 2024-12-04T14:59:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T14:59:31,934 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=801, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/bf76421f46b04dc3ae7d49d7bae0ff18 2024-12-04T14:59:31,948 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/e20ab1a437124d1dab2a138521d6837c is 50, key is test_row_0/B:col10/1733324370931/Put/seqid=0 2024-12-04T14:59:31,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742048_1224 (size=12301) 2024-12-04T14:59:32,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T14:59:32,353 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=801 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/e20ab1a437124d1dab2a138521d6837c 2024-12-04T14:59:32,368 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/a0139a785fbb4c86a044a0975dbfb852 is 50, key is test_row_0/C:col10/1733324370931/Put/seqid=0 2024-12-04T14:59:32,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742049_1225 (size=12301) 2024-12-04T14:59:32,773 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=801 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/a0139a785fbb4c86a044a0975dbfb852 2024-12-04T14:59:32,778 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/A/bf76421f46b04dc3ae7d49d7bae0ff18 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/bf76421f46b04dc3ae7d49d7bae0ff18 2024-12-04T14:59:32,784 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/bf76421f46b04dc3ae7d49d7bae0ff18, entries=150, sequenceid=801, filesize=30.5 K 2024-12-04T14:59:32,785 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/B/e20ab1a437124d1dab2a138521d6837c as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/e20ab1a437124d1dab2a138521d6837c 2024-12-04T14:59:32,790 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/e20ab1a437124d1dab2a138521d6837c, entries=150, sequenceid=801, filesize=12.0 K 2024-12-04T14:59:32,790 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/.tmp/C/a0139a785fbb4c86a044a0975dbfb852 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/a0139a785fbb4c86a044a0975dbfb852 2024-12-04T14:59:32,794 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/a0139a785fbb4c86a044a0975dbfb852, entries=150, sequenceid=801, filesize=12.0 K 2024-12-04T14:59:32,795 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 47aad6a6d662b77a47b7fe8495dd267c in 1691ms, sequenceid=801, compaction requested=true 2024-12-04T14:59:32,795 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/776ddd523f0748da9eb69c5cd6e202f5, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/050fe7463e73437894b5a69e45082d7f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/372010f4b8f747cd810afd586d5a77b5, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/77a886794d1246afb53c20109b74bf31, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/4e27dd90e2234a4d9e020d1ad486b53d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6df324351c3945368c42193cc07f10f5, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/2ca2263f6681429a8b6a208ec4cb7f9c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6a70daf6658c4896a813d394c69d5af8, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/7d6bec5c0c3f41fbae4059e42b21971e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/14af9a4170a5454dac525e8f3e5157a2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6b2f76428d48496eb053765985ad9cf3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/1f4c20ba607d4d6ca184112392497770, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/d9469c995c084596948281fa62b27df9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/ff8757d3827943968a3f17688ce2bd11, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/d7588cd953c24db4a05d9c0dedee6c9a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/2b88cc3f5dcc4a68937d8dff453b633f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/f36db39a041848a19f81d651d4c93690, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/76e0b95e0a7c40c78b32c1c95dea7a40, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/ace0f9c49ce6421db84993688bb91411, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/430694f4a842435baf9f399ba2986805, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/fde6ad8fdd7340efa37d20dd1bf03109, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/0649ee88031748f385ba914e60e407de, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/cbd82a5f708446d4a70d1ae518c2531f] to archive 2024-12-04T14:59:32,796 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:59:32,798 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/776ddd523f0748da9eb69c5cd6e202f5 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/776ddd523f0748da9eb69c5cd6e202f5 2024-12-04T14:59:32,799 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/050fe7463e73437894b5a69e45082d7f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/050fe7463e73437894b5a69e45082d7f 2024-12-04T14:59:32,800 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/372010f4b8f747cd810afd586d5a77b5 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/372010f4b8f747cd810afd586d5a77b5 2024-12-04T14:59:32,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/77a886794d1246afb53c20109b74bf31 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/77a886794d1246afb53c20109b74bf31 2024-12-04T14:59:32,802 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/4e27dd90e2234a4d9e020d1ad486b53d to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/4e27dd90e2234a4d9e020d1ad486b53d 2024-12-04T14:59:32,802 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6df324351c3945368c42193cc07f10f5 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6df324351c3945368c42193cc07f10f5 2024-12-04T14:59:32,804 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/2ca2263f6681429a8b6a208ec4cb7f9c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/2ca2263f6681429a8b6a208ec4cb7f9c 2024-12-04T14:59:32,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6a70daf6658c4896a813d394c69d5af8 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6a70daf6658c4896a813d394c69d5af8 2024-12-04T14:59:32,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/7d6bec5c0c3f41fbae4059e42b21971e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/7d6bec5c0c3f41fbae4059e42b21971e 2024-12-04T14:59:32,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/14af9a4170a5454dac525e8f3e5157a2 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/14af9a4170a5454dac525e8f3e5157a2 2024-12-04T14:59:32,807 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6b2f76428d48496eb053765985ad9cf3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6b2f76428d48496eb053765985ad9cf3 2024-12-04T14:59:32,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/1f4c20ba607d4d6ca184112392497770 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/1f4c20ba607d4d6ca184112392497770 2024-12-04T14:59:32,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/d9469c995c084596948281fa62b27df9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/d9469c995c084596948281fa62b27df9 2024-12-04T14:59:32,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/ff8757d3827943968a3f17688ce2bd11 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/ff8757d3827943968a3f17688ce2bd11 2024-12-04T14:59:32,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/d7588cd953c24db4a05d9c0dedee6c9a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/d7588cd953c24db4a05d9c0dedee6c9a 2024-12-04T14:59:32,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/2b88cc3f5dcc4a68937d8dff453b633f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/2b88cc3f5dcc4a68937d8dff453b633f 2024-12-04T14:59:32,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/f36db39a041848a19f81d651d4c93690 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/f36db39a041848a19f81d651d4c93690 2024-12-04T14:59:32,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/76e0b95e0a7c40c78b32c1c95dea7a40 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/76e0b95e0a7c40c78b32c1c95dea7a40 2024-12-04T14:59:32,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/ace0f9c49ce6421db84993688bb91411 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/ace0f9c49ce6421db84993688bb91411 2024-12-04T14:59:32,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/430694f4a842435baf9f399ba2986805 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/430694f4a842435baf9f399ba2986805 2024-12-04T14:59:32,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/fde6ad8fdd7340efa37d20dd1bf03109 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/fde6ad8fdd7340efa37d20dd1bf03109 2024-12-04T14:59:32,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/0649ee88031748f385ba914e60e407de to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/0649ee88031748f385ba914e60e407de 2024-12-04T14:59:32,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/cbd82a5f708446d4a70d1ae518c2531f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/cbd82a5f708446d4a70d1ae518c2531f 2024-12-04T14:59:32,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/3a505fc8e0834ea4a29b9d555b9f8c10, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/526d2ad5271743e58060da0d08d83c9a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/f602ddc54f85457fae78dc3b64f137c7, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/64d222d9b97e42aebddcf8f84d87b09e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/88b18307c7b349498eae0821e4000a1d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/4873b4aa1ed649a58debcb7e16adc1ed, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/e3e053773bbe4120a5f7684be7a2f99c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/662b1f4bd7274c0f8940d5da872755f8, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/b5dd80edf3b44c9e87f99720db9b3137, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/2c9394aae445466bbf02ecdc6bcb05b2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/969bfe2fb47d43bea8dcb3c50d75f7eb, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/446c89a74831498297bf7d8565114c94, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/b8c1fd0403574008b133fbaf09a2ac20, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/428196bb466e4eac97e17b4bd57fc1f7, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/8f10176d8b94472eb2464d62843eb266, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/fc027cfd580249dfa0943d23daadd0e9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/75dfe4e1d28e4208a7b3b6f523679aa9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/1faffb4e08094e7d970d919f705bf024, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/b71c1d4756bc4fbd8f2dccf117074964, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/c1a4df41d0b94b228c168a1cc7f6362d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/38c31d6fa66f4a18a46bbd233c127083, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/f07bf3f57e75471c9da0b76c22235d27, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/e43ddca178094e00b24442df7020d49a] to archive 2024-12-04T14:59:32,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:59:32,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/3a505fc8e0834ea4a29b9d555b9f8c10 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/3a505fc8e0834ea4a29b9d555b9f8c10 2024-12-04T14:59:32,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/526d2ad5271743e58060da0d08d83c9a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/526d2ad5271743e58060da0d08d83c9a 2024-12-04T14:59:32,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/f602ddc54f85457fae78dc3b64f137c7 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/f602ddc54f85457fae78dc3b64f137c7 2024-12-04T14:59:32,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/64d222d9b97e42aebddcf8f84d87b09e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/64d222d9b97e42aebddcf8f84d87b09e 2024-12-04T14:59:32,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/88b18307c7b349498eae0821e4000a1d to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/88b18307c7b349498eae0821e4000a1d 2024-12-04T14:59:32,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/4873b4aa1ed649a58debcb7e16adc1ed to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/4873b4aa1ed649a58debcb7e16adc1ed 2024-12-04T14:59:32,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/e3e053773bbe4120a5f7684be7a2f99c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/e3e053773bbe4120a5f7684be7a2f99c 2024-12-04T14:59:32,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/662b1f4bd7274c0f8940d5da872755f8 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/662b1f4bd7274c0f8940d5da872755f8 2024-12-04T14:59:32,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/b5dd80edf3b44c9e87f99720db9b3137 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/b5dd80edf3b44c9e87f99720db9b3137 2024-12-04T14:59:32,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/2c9394aae445466bbf02ecdc6bcb05b2 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/2c9394aae445466bbf02ecdc6bcb05b2 2024-12-04T14:59:32,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/969bfe2fb47d43bea8dcb3c50d75f7eb to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/969bfe2fb47d43bea8dcb3c50d75f7eb 2024-12-04T14:59:32,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/446c89a74831498297bf7d8565114c94 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/446c89a74831498297bf7d8565114c94 2024-12-04T14:59:32,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/b8c1fd0403574008b133fbaf09a2ac20 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/b8c1fd0403574008b133fbaf09a2ac20 2024-12-04T14:59:32,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/428196bb466e4eac97e17b4bd57fc1f7 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/428196bb466e4eac97e17b4bd57fc1f7 2024-12-04T14:59:32,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/8f10176d8b94472eb2464d62843eb266 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/8f10176d8b94472eb2464d62843eb266 2024-12-04T14:59:32,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/fc027cfd580249dfa0943d23daadd0e9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/fc027cfd580249dfa0943d23daadd0e9 2024-12-04T14:59:32,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/75dfe4e1d28e4208a7b3b6f523679aa9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/75dfe4e1d28e4208a7b3b6f523679aa9 2024-12-04T14:59:32,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/1faffb4e08094e7d970d919f705bf024 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/1faffb4e08094e7d970d919f705bf024 2024-12-04T14:59:32,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/b71c1d4756bc4fbd8f2dccf117074964 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/b71c1d4756bc4fbd8f2dccf117074964 2024-12-04T14:59:32,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/c1a4df41d0b94b228c168a1cc7f6362d to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/c1a4df41d0b94b228c168a1cc7f6362d 2024-12-04T14:59:32,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/38c31d6fa66f4a18a46bbd233c127083 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/38c31d6fa66f4a18a46bbd233c127083 2024-12-04T14:59:32,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/f07bf3f57e75471c9da0b76c22235d27 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/f07bf3f57e75471c9da0b76c22235d27 2024-12-04T14:59:32,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/e43ddca178094e00b24442df7020d49a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/e43ddca178094e00b24442df7020d49a 2024-12-04T14:59:32,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/c01e14c081c04805bf4ab65718927b20, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/13fbad04d2ef4b2cbf9d040695385791, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2aa7a430ca5e46a5b6c4a51766d6976b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/a0970d07d1304bd1bf43a165ee569ea2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/96ea025439bd45b88b452bac6536bc08, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/52ef8a602a664cbcac6d791606e3e685, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/735d255f8a594061a3329ffbdc5f0f1e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/d8b85c363c904dcc8eb9e61761abb8c0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/fbf0e52a597b4c169ecd6ef1931ccb85, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/257cc7c7d4ff46d1b67d59edce8025bf, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/bee8b8cbe1f3491e89047ea67cf2f4ee, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/fae8c59f7fd34df59250bbf981129fae, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/d7db5ce70800493989ecb047663e21d3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2ca9d33ac4114d5da443f6fade5d7db7, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/b871b286d4d046c2b4bc88dcde04886a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/919a3e7a180940ecbca41e15aa53004b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/5e002d4423844596866b2610b51e3b7e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/11a1c63684a34abebbc7151dd70e9467, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/e2819870a3494adf95ef0031229f5c3f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2b61183d3cff4548b91ec0ca555923b7, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2ae858b3abab468e9fa8996309df90b4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/6c741c02a1494f468d2ab47ebe6ab615, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/0a8b3aa64f61495897e8d86f74435a59] to archive 2024-12-04T14:59:32,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:59:32,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/c01e14c081c04805bf4ab65718927b20 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/c01e14c081c04805bf4ab65718927b20 2024-12-04T14:59:32,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/13fbad04d2ef4b2cbf9d040695385791 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/13fbad04d2ef4b2cbf9d040695385791 2024-12-04T14:59:32,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2aa7a430ca5e46a5b6c4a51766d6976b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2aa7a430ca5e46a5b6c4a51766d6976b 2024-12-04T14:59:32,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/a0970d07d1304bd1bf43a165ee569ea2 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/a0970d07d1304bd1bf43a165ee569ea2 2024-12-04T14:59:32,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/96ea025439bd45b88b452bac6536bc08 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/96ea025439bd45b88b452bac6536bc08 2024-12-04T14:59:32,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/52ef8a602a664cbcac6d791606e3e685 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/52ef8a602a664cbcac6d791606e3e685 2024-12-04T14:59:32,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/735d255f8a594061a3329ffbdc5f0f1e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/735d255f8a594061a3329ffbdc5f0f1e 2024-12-04T14:59:32,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/d8b85c363c904dcc8eb9e61761abb8c0 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/d8b85c363c904dcc8eb9e61761abb8c0 2024-12-04T14:59:32,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/fbf0e52a597b4c169ecd6ef1931ccb85 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/fbf0e52a597b4c169ecd6ef1931ccb85 2024-12-04T14:59:32,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/257cc7c7d4ff46d1b67d59edce8025bf to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/257cc7c7d4ff46d1b67d59edce8025bf 2024-12-04T14:59:32,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/bee8b8cbe1f3491e89047ea67cf2f4ee to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/bee8b8cbe1f3491e89047ea67cf2f4ee 2024-12-04T14:59:32,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/fae8c59f7fd34df59250bbf981129fae to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/fae8c59f7fd34df59250bbf981129fae 2024-12-04T14:59:32,869 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/d7db5ce70800493989ecb047663e21d3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/d7db5ce70800493989ecb047663e21d3 2024-12-04T14:59:32,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2ca9d33ac4114d5da443f6fade5d7db7 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2ca9d33ac4114d5da443f6fade5d7db7 2024-12-04T14:59:32,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/b871b286d4d046c2b4bc88dcde04886a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/b871b286d4d046c2b4bc88dcde04886a 2024-12-04T14:59:32,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/919a3e7a180940ecbca41e15aa53004b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/919a3e7a180940ecbca41e15aa53004b 2024-12-04T14:59:32,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/5e002d4423844596866b2610b51e3b7e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/5e002d4423844596866b2610b51e3b7e 2024-12-04T14:59:32,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/11a1c63684a34abebbc7151dd70e9467 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/11a1c63684a34abebbc7151dd70e9467 2024-12-04T14:59:32,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/e2819870a3494adf95ef0031229f5c3f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/e2819870a3494adf95ef0031229f5c3f 2024-12-04T14:59:32,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2b61183d3cff4548b91ec0ca555923b7 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2b61183d3cff4548b91ec0ca555923b7 2024-12-04T14:59:32,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2ae858b3abab468e9fa8996309df90b4 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2ae858b3abab468e9fa8996309df90b4 2024-12-04T14:59:32,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/6c741c02a1494f468d2ab47ebe6ab615 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/6c741c02a1494f468d2ab47ebe6ab615 2024-12-04T14:59:32,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/0a8b3aa64f61495897e8d86f74435a59 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/0a8b3aa64f61495897e8d86f74435a59 2024-12-04T14:59:32,880 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/recovered.edits/804.seqid, newMaxSeqId=804, maxSeqId=4 2024-12-04T14:59:32,881 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. 2024-12-04T14:59:32,881 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1676): Region close journal for 47aad6a6d662b77a47b7fe8495dd267c: Waiting for close lock at 1733324371104Running coprocessor pre-close hooks at 1733324371104Disabling compacts and flushes for region at 1733324371104Disabling writes for close at 1733324371104Obtaining lock to block concurrent updates at 1733324371105 (+1 ms)Preparing flush snapshotting stores in 47aad6a6d662b77a47b7fe8495dd267c at 1733324371105Finished memstore snapshotting TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733324371106 (+1 ms)Flushing stores of TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c. at 1733324371107 (+1 ms)Flushing 47aad6a6d662b77a47b7fe8495dd267c/A: creating writer at 1733324371107Flushing mob file 47aad6a6d662b77a47b7fe8495dd267c/A: appending metadata at 1733324371116 (+9 ms)Flushing mob file 47aad6a6d662b77a47b7fe8495dd267c/A: closing flushed file at 1733324371116Flushing 47aad6a6d662b77a47b7fe8495dd267c/A: appending metadata at 1733324371528 (+412 ms)Flushing 47aad6a6d662b77a47b7fe8495dd267c/A: closing flushed file at 1733324371529 (+1 ms)Flushing 47aad6a6d662b77a47b7fe8495dd267c/B: creating writer at 1733324371943 (+414 ms)Flushing 47aad6a6d662b77a47b7fe8495dd267c/B: appending metadata at 1733324371947 (+4 ms)Flushing 47aad6a6d662b77a47b7fe8495dd267c/B: closing flushed file at 1733324371947Flushing 47aad6a6d662b77a47b7fe8495dd267c/C: creating writer at 1733324372364 (+417 ms)Flushing 47aad6a6d662b77a47b7fe8495dd267c/C: appending metadata at 1733324372368 (+4 ms)Flushing 47aad6a6d662b77a47b7fe8495dd267c/C: closing flushed file at 1733324372368Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fba8a35: reopening flushed file at 1733324372777 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d2ba63a: reopening flushed file at 1733324372784 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f1f9e26: reopening flushed file at 1733324372790 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 47aad6a6d662b77a47b7fe8495dd267c in 1691ms, sequenceid=801, compaction requested=true at 1733324372795 (+5 ms)Writing region close event to WAL at 1733324372878 (+83 ms)Running coprocessor post-close hooks at 1733324372881 (+3 ms)Closed at 1733324372881 2024-12-04T14:59:32,883 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(157): Closed 47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:32,883 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=47aad6a6d662b77a47b7fe8495dd267c, regionState=CLOSED 2024-12-04T14:59:32,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 because future has completed 2024-12-04T14:59:32,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-04T14:59:32,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseRegionProcedure 47aad6a6d662b77a47b7fe8495dd267c, server=d214edae7bd2,36649,1733324304801 in 1.9350 sec 2024-12-04T14:59:32,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-12-04T14:59:32,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47aad6a6d662b77a47b7fe8495dd267c, UNASSIGN in 1.9390 sec 2024-12-04T14:59:32,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-04T14:59:32,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9420 sec 2024-12-04T14:59:32,892 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324372891"}]},"ts":"1733324372891"} 2024-12-04T14:59:32,893 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T14:59:32,893 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T14:59:32,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9560 sec 2024-12-04T14:59:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T14:59:33,091 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:33,091 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T14:59:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:59:33,093 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:59:33,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T14:59:33,093 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=54, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:59:33,097 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,101 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A, FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B, FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C, FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/recovered.edits] 2024-12-04T14:59:33,105 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/36aa448620504b4c867900c1e4a8b7ab to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/36aa448620504b4c867900c1e4a8b7ab 2024-12-04T14:59:33,107 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6a785c0e7a5a4c8a9069ca77f79aae24 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/6a785c0e7a5a4c8a9069ca77f79aae24 2024-12-04T14:59:33,108 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/bf76421f46b04dc3ae7d49d7bae0ff18 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/bf76421f46b04dc3ae7d49d7bae0ff18 2024-12-04T14:59:33,110 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/fc51889d71404473bf6873137b6bd13b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/A/fc51889d71404473bf6873137b6bd13b 2024-12-04T14:59:33,113 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/38d21b2481a9466692f1595935b55de6 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/38d21b2481a9466692f1595935b55de6 2024-12-04T14:59:33,115 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/4d6cc26be55e4837a45015311302ccd9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/4d6cc26be55e4837a45015311302ccd9 2024-12-04T14:59:33,117 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/caabf27702c0435c802309cb75ecc31a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/caabf27702c0435c802309cb75ecc31a 2024-12-04T14:59:33,119 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/e20ab1a437124d1dab2a138521d6837c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/B/e20ab1a437124d1dab2a138521d6837c 2024-12-04T14:59:33,122 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2e0125766821421c93f5bbe5c41d3eb6 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/2e0125766821421c93f5bbe5c41d3eb6 2024-12-04T14:59:33,124 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/77c05b533c2448d482aa8e022bea95ce to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/77c05b533c2448d482aa8e022bea95ce 2024-12-04T14:59:33,125 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/a0139a785fbb4c86a044a0975dbfb852 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/a0139a785fbb4c86a044a0975dbfb852 2024-12-04T14:59:33,127 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/a0223d0bdc9248818884392da0d5eb82 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/C/a0223d0bdc9248818884392da0d5eb82 2024-12-04T14:59:33,131 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/recovered.edits/804.seqid to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c/recovered.edits/804.seqid 2024-12-04T14:59:33,132 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,132 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T14:59:33,132 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T14:59:33,133 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-04T14:59:33,139 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040c12183f57f1415ba0a4e3e5220995eb_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040c12183f57f1415ba0a4e3e5220995eb_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,141 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040db90efbf7ce45fda83a930f60a3ceb5_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040db90efbf7ce45fda83a930f60a3ceb5_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,143 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041640fe523f43401c831fdd1d8e81682f_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041640fe523f43401c831fdd1d8e81682f_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,145 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041a73387e85a8417ea47007afac78dcf2_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041a73387e85a8417ea47007afac78dcf2_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,147 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204218aaac014dc4affbac5b1e7124fad05_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204218aaac014dc4affbac5b1e7124fad05_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,149 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412044ac74d3932b84bae92494c66139899c3_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412044ac74d3932b84bae92494c66139899c3_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,151 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412044f905bb204ca41fda7c4fc54213f4fd7_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412044f905bb204ca41fda7c4fc54213f4fd7_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,153 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120453922ade24b742b5b21e2d6b9ca91a0f_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120453922ade24b742b5b21e2d6b9ca91a0f_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,155 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120455b8c95f4e424963bc7e44c8f90e8e16_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120455b8c95f4e424963bc7e44c8f90e8e16_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,157 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120456724c33c0c2497a867f0b2a0e212e7b_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120456724c33c0c2497a867f0b2a0e212e7b_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,159 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046cc1a7d8fa4d454b9ca520db6e896caf_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046cc1a7d8fa4d454b9ca520db6e896caf_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,160 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046ec54a6e55244552b5a69944c78543f2_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046ec54a6e55244552b5a69944c78543f2_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,161 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048326eb7559ff43d989880adc96473588_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048326eb7559ff43d989880adc96473588_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,162 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120487c303a5634c4af5bec6485bdb00bb95_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120487c303a5634c4af5bec6485bdb00bb95_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,164 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204aed5bca425a94a309f81f9328a151b2a_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204aed5bca425a94a309f81f9328a151b2a_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,164 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c6b569d705254b2190af9fb072f6f9fc_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c6b569d705254b2190af9fb072f6f9fc_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,166 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204cc7975a938d04312b225f36a6d723775_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204cc7975a938d04312b225f36a6d723775_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,167 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d695c84c51f54487983dab9ade1c4c1c_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d695c84c51f54487983dab9ade1c4c1c_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,168 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e22bb65cc71f4829b6663e898579b92a_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e22bb65cc71f4829b6663e898579b92a_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,169 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ed6b6bd95f2744ebbf1ece9ab534747d_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ed6b6bd95f2744ebbf1ece9ab534747d_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,169 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f2436ea3141c4214a7111264844f729c_47aad6a6d662b77a47b7fe8495dd267c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f2436ea3141c4214a7111264844f729c_47aad6a6d662b77a47b7fe8495dd267c 2024-12-04T14:59:33,170 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T14:59:33,172 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=54, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:59:33,174 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T14:59:33,176 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T14:59:33,177 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=54, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:59:33,177 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T14:59:33,177 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733324373177"}]},"ts":"9223372036854775807"} 2024-12-04T14:59:33,179 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T14:59:33,179 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 47aad6a6d662b77a47b7fe8495dd267c, NAME => 'TestAcidGuarantees,,1733324337572.47aad6a6d662b77a47b7fe8495dd267c.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T14:59:33,179 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T14:59:33,180 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733324373179"}]},"ts":"9223372036854775807"} 2024-12-04T14:59:33,181 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T14:59:33,182 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=54, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:59:33,183 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 91 msec 2024-12-04T14:59:33,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T14:59:33,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T14:59:33,201 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:33,208 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 249), OpenFileDescriptor=451 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=597 (was 661), ProcessCount=11 (was 11), AvailableMemoryMB=9265 (was 8920) - AvailableMemoryMB LEAK? - 2024-12-04T14:59:33,216 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=597, ProcessCount=11, AvailableMemoryMB=9265 2024-12-04T14:59:33,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T14:59:33,217 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:59:33,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T14:59:33,219 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T14:59:33,219 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:33,219 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 55 2024-12-04T14:59:33,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-04T14:59:33,220 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T14:59:33,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742050_1226 (size=1061) 2024-12-04T14:59:33,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-04T14:59:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-04T14:59:33,629 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee 2024-12-04T14:59:33,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742051_1227 (size=53) 2024-12-04T14:59:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-04T14:59:34,039 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:59:34,039 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cf9bed3c8089ea388e8abd4912312aee, disabling compactions & flushes 2024-12-04T14:59:34,039 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:34,039 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:34,039 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. after waiting 0 ms 2024-12-04T14:59:34,039 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:34,039 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:34,039 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cf9bed3c8089ea388e8abd4912312aee: Waiting for close lock at 1733324374039Disabling compacts and flushes for region at 1733324374039Disabling writes for close at 1733324374039Writing region close event to WAL at 1733324374039Closed at 1733324374039 2024-12-04T14:59:34,040 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T14:59:34,040 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733324374040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733324374040"}]},"ts":"1733324374040"} 2024-12-04T14:59:34,041 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T14:59:34,042 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T14:59:34,042 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324374042"}]},"ts":"1733324374042"} 2024-12-04T14:59:34,043 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T14:59:34,043 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf9bed3c8089ea388e8abd4912312aee, ASSIGN}] 2024-12-04T14:59:34,044 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf9bed3c8089ea388e8abd4912312aee, ASSIGN 2024-12-04T14:59:34,045 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf9bed3c8089ea388e8abd4912312aee, ASSIGN; state=OFFLINE, location=d214edae7bd2,36649,1733324304801; forceNewPlan=false, retain=false 2024-12-04T14:59:34,195 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=cf9bed3c8089ea388e8abd4912312aee, regionState=OPENING, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:34,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf9bed3c8089ea388e8abd4912312aee, ASSIGN because future has completed 2024-12-04T14:59:34,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801}] 2024-12-04T14:59:34,353 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:34,353 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7752): Opening region: {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:59:34,353 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:34,353 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:59:34,353 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7794): checking encryption for cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:34,353 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7797): checking classloading for cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:34,355 INFO [StoreOpener-cf9bed3c8089ea388e8abd4912312aee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:34,356 INFO [StoreOpener-cf9bed3c8089ea388e8abd4912312aee-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T14:59:34,356 INFO [StoreOpener-cf9bed3c8089ea388e8abd4912312aee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf9bed3c8089ea388e8abd4912312aee columnFamilyName A 2024-12-04T14:59:34,356 DEBUG [StoreOpener-cf9bed3c8089ea388e8abd4912312aee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:34,357 INFO [StoreOpener-cf9bed3c8089ea388e8abd4912312aee-1 {}] regionserver.HStore(327): Store=cf9bed3c8089ea388e8abd4912312aee/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:59:34,357 INFO [StoreOpener-cf9bed3c8089ea388e8abd4912312aee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:34,358 INFO [StoreOpener-cf9bed3c8089ea388e8abd4912312aee-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T14:59:34,358 INFO [StoreOpener-cf9bed3c8089ea388e8abd4912312aee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf9bed3c8089ea388e8abd4912312aee columnFamilyName B 2024-12-04T14:59:34,358 DEBUG [StoreOpener-cf9bed3c8089ea388e8abd4912312aee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:34,359 INFO [StoreOpener-cf9bed3c8089ea388e8abd4912312aee-1 {}] regionserver.HStore(327): Store=cf9bed3c8089ea388e8abd4912312aee/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:59:34,359 INFO [StoreOpener-cf9bed3c8089ea388e8abd4912312aee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:34,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-04T14:59:34,360 INFO [StoreOpener-cf9bed3c8089ea388e8abd4912312aee-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T14:59:34,361 INFO [StoreOpener-cf9bed3c8089ea388e8abd4912312aee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf9bed3c8089ea388e8abd4912312aee columnFamilyName C 2024-12-04T14:59:34,361 DEBUG [StoreOpener-cf9bed3c8089ea388e8abd4912312aee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:34,361 INFO [StoreOpener-cf9bed3c8089ea388e8abd4912312aee-1 {}] regionserver.HStore(327): Store=cf9bed3c8089ea388e8abd4912312aee/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:59:34,361 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:34,361 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1038): replaying wal for cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:34,362 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:34,362 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:34,363 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1048): stopping wal replay for cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:34,363 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1060): Cleaning up temporary data for cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:34,364 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:59:34,365 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1093): writing seq id for cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:34,367 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:59:34,367 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1114): Opened cf9bed3c8089ea388e8abd4912312aee; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68095070, jitterRate=0.014695614576339722}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:59:34,367 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:34,368 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1006): Region open journal for cf9bed3c8089ea388e8abd4912312aee: Running coprocessor pre-open hook at 1733324374353Writing region info on filesystem at 1733324374353Initializing all the Stores at 1733324374354 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324374354Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324374355 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324374355Cleaning up temporary data from old regions at 1733324374363 (+8 ms)Running coprocessor post-open hooks at 1733324374367 (+4 ms)Region opened successfully at 1733324374368 (+1 ms) 2024-12-04T14:59:34,369 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., pid=57, masterSystemTime=1733324374350 2024-12-04T14:59:34,370 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:34,370 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:34,371 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=cf9bed3c8089ea388e8abd4912312aee, regionState=OPEN, openSeqNum=2, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:34,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 because future has completed 2024-12-04T14:59:34,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-04T14:59:34,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; OpenRegionProcedure cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 in 176 msec 2024-12-04T14:59:34,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-12-04T14:59:34,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf9bed3c8089ea388e8abd4912312aee, ASSIGN in 331 msec 2024-12-04T14:59:34,377 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T14:59:34,377 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324374377"}]},"ts":"1733324374377"} 2024-12-04T14:59:34,379 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T14:59:34,380 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T14:59:34,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-12-04T14:59:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-04T14:59:35,371 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:35,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ce28163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,373 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:59:35,373 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:59:35,375 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:59:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:59:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:59:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b81cba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:59:35,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:59:35,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:35,378 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:59:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc25202, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,379 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:59:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:59:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:59:35,381 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:59:35,382 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T14:59:35,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is d214edae7bd2,39449,1733324304115 2024-12-04T14:59:35,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@59b47730 2024-12-04T14:59:35,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T14:59:35,385 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T14:59:35,386 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7642a08c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,386 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:59:35,386 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:59:35,387 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:59:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:59:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:59:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728bcc96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:59:35,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:59:35,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:35,388 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:59:35,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b7083cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ecfb794, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,390 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:59:35,390 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:59:35,390 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:59:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:59:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:59:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b19ac5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:59:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:59:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:35,392 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:59:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2447a01d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff0f448, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,393 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:59:35,393 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:59:35,394 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:59:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:59:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:59:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74b2680, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:59:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:59:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:35,395 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:59:35,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c64f8e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d560643, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,396 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:59:35,396 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:59:35,397 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:59:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:59:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:59:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d1bc829, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:59:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:59:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:35,398 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51072, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:59:35,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af30f67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6509bdf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,399 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:59:35,400 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:59:35,400 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:59:35,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:59:35,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:59:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75639166, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:59:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:59:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:35,402 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:59:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62387c2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@beb69b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,403 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:59:35,403 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:59:35,404 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:59:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:59:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:59:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fc264ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:59:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:59:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:35,406 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:59:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ceabf9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643da6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,407 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:59:35,407 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:59:35,408 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:59:35,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:59:35,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:59:35,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f556a93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:59:35,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:59:35,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:35,409 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:59:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@288cbe53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4829bcb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,410 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:59:35,410 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:59:35,411 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:59:35,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:59:35,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:59:35,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5002b38b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:59:35,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:59:35,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:35,412 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:59:35,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52224342, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d96952, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,413 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:59:35,413 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:59:35,414 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:59:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:59:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:59:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e431a82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:59:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:59:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:35,416 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:59:35,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78178b68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24595b5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,417 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T14:59:35,417 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:59:35,418 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T14:59:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:59:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T14:59:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a6fc3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T14:59:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:59:35,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:35,419 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:59:35,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57113f61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:59:35,423 DEBUG [Thread-1075 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:59:35,423 DEBUG [Thread-1073 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:59:35,423 DEBUG [Thread-1069 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:59:35,423 DEBUG [Thread-1077 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:59:35,423 DEBUG [Thread-1088 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:59:35,423 DEBUG [Thread-1071 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:59:35,424 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:59:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:59:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:59:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:59:35,424 DEBUG [Thread-1082 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:59:35,425 DEBUG [Thread-1084 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:59:35,425 DEBUG [Thread-1080 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:59:35,425 DEBUG [Thread-1086 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:59:35,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:59:35,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-12-04T14:59:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:59:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:59:35,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:59:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:59:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:59:35,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:59:35,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:59:35,427 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:59:35,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:59:35,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:59:35,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:59:35,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:59:35,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:59:35,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:59:35,428 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:59:35,429 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:59:35,429 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:59:35,429 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:59:35,430 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:59:35,430 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:59:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:59:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:59:35,430 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:59:35,430 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:59:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T14:59:35,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:59:35,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T14:59:35,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T14:59:35,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T14:59:35,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T14:59:35,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T14:59:35,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:59:35,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T14:59:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T14:59:35,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T14:59:35,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:59:35,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T14:59:35,437 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:59:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T14:59:35,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T14:59:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:35,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T14:59:35,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:35,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:35,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:35,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:35,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:35,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:35,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/2e36d62ec39e45cabaeadd8efa4c1fe4 is 50, key is test_row_0/A:col10/1733324375432/Put/seqid=0 2024-12-04T14:59:35,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:35,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324435467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:35,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:35,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324375458, max timestamp=1733324375476Num uniques 859; 2024-12-04T14:59:35,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:35,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324375458, max timestamp=1733324375476Num uniques 859; 2024-12-04T14:59:35,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:35,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324375458, max timestamp=1733324375476Num uniques 859; 2024-12-04T14:59:35,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:35,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742052_1228 (size=12001) 2024-12-04T14:59:35,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324435505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324435506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:35,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324435508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:35,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324435511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T14:59:35,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-04T14:59:35,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:35,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:35,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:35,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:35,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:35,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324435590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:35,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:35,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324435620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:35,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324435621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:35,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324435621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:35,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324435621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:35,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:35,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:35,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-04T14:59:35,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:35,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:35,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:35,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:35,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T14:59:35,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324435810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:35,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:35,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324435830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:35,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:35,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324435830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:35,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:35,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324435830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:35,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:35,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324435831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:35,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:35,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-04T14:59:35,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:35,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:35,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:35,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:35,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:35,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/2e36d62ec39e45cabaeadd8efa4c1fe4 2024-12-04T14:59:35,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/3821ef0d222647cf8e64cf8f44606b4b is 50, key is test_row_0/B:col10/1733324375432/Put/seqid=0 2024-12-04T14:59:35,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742053_1229 (size=12001) 2024-12-04T14:59:36,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-04T14:59:36,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:36,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:36,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:36,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:36,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:36,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T14:59:36,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324436120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:36,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:36,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:36,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324436140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:36,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324436140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:36,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324436140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:36,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324436140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:36,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-04T14:59:36,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:36,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:36,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:36,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:36,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:36,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/3821ef0d222647cf8e64cf8f44606b4b 2024-12-04T14:59:36,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-04T14:59:36,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:36,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:36,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:36,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:36,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:36,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/306f1512844d4026aaf0cb8688516868 is 50, key is test_row_0/C:col10/1733324375432/Put/seqid=0 2024-12-04T14:59:36,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742054_1230 (size=12001) 2024-12-04T14:59:36,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-04T14:59:36,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:36,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:36,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:36,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:36,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T14:59:36,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324436630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:36,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324436651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:36,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324436651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:36,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324436651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:36,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324436651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:36,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-04T14:59:36,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:36,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:36,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:36,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:36,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:36,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/306f1512844d4026aaf0cb8688516868 2024-12-04T14:59:36,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/2e36d62ec39e45cabaeadd8efa4c1fe4 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/2e36d62ec39e45cabaeadd8efa4c1fe4 2024-12-04T14:59:36,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/2e36d62ec39e45cabaeadd8efa4c1fe4, entries=150, sequenceid=12, filesize=11.7 K 2024-12-04T14:59:36,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/3821ef0d222647cf8e64cf8f44606b4b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/3821ef0d222647cf8e64cf8f44606b4b 2024-12-04T14:59:36,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/3821ef0d222647cf8e64cf8f44606b4b, entries=150, sequenceid=12, filesize=11.7 K 2024-12-04T14:59:36,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/306f1512844d4026aaf0cb8688516868 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/306f1512844d4026aaf0cb8688516868 2024-12-04T14:59:36,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-04T14:59:36,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:36,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:36,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:36,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:36,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:36,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/306f1512844d4026aaf0cb8688516868, entries=150, sequenceid=12, filesize=11.7 K 2024-12-04T14:59:36,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cf9bed3c8089ea388e8abd4912312aee in 1378ms, sequenceid=12, compaction requested=false 2024-12-04T14:59:36,819 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-04T14:59:36,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:36,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-04T14:59:36,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:36,961 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-04T14:59:36,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:36,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:36,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:36,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:36,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:36,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:36,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/b98c0fb81b0a4ae28d3fa5d4a401a6db is 50, key is test_row_0/A:col10/1733324375504/Put/seqid=0 2024-12-04T14:59:36,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742055_1231 (size=12001) 2024-12-04T14:59:36,985 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/b98c0fb81b0a4ae28d3fa5d4a401a6db 2024-12-04T14:59:37,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/d175a81cfc8f44529c1eb682adb4e370 is 50, key is test_row_0/B:col10/1733324375504/Put/seqid=0 2024-12-04T14:59:37,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742056_1232 (size=12001) 2024-12-04T14:59:37,420 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/d175a81cfc8f44529c1eb682adb4e370 2024-12-04T14:59:37,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/464c6bab6cad4169945f35d6b066d622 is 50, key is test_row_0/C:col10/1733324375504/Put/seqid=0 2024-12-04T14:59:37,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742057_1233 (size=12001) 2024-12-04T14:59:37,445 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/464c6bab6cad4169945f35d6b066d622 2024-12-04T14:59:37,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/b98c0fb81b0a4ae28d3fa5d4a401a6db as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/b98c0fb81b0a4ae28d3fa5d4a401a6db 2024-12-04T14:59:37,456 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/b98c0fb81b0a4ae28d3fa5d4a401a6db, entries=150, sequenceid=47, filesize=11.7 K 2024-12-04T14:59:37,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/d175a81cfc8f44529c1eb682adb4e370 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/d175a81cfc8f44529c1eb682adb4e370 2024-12-04T14:59:37,463 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/d175a81cfc8f44529c1eb682adb4e370, entries=150, sequenceid=47, filesize=11.7 K 2024-12-04T14:59:37,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/464c6bab6cad4169945f35d6b066d622 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/464c6bab6cad4169945f35d6b066d622 2024-12-04T14:59:37,474 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/464c6bab6cad4169945f35d6b066d622, entries=150, sequenceid=47, filesize=11.7 K 2024-12-04T14:59:37,476 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=0 B/0 for cf9bed3c8089ea388e8abd4912312aee in 515ms, sequenceid=47, compaction requested=false 2024-12-04T14:59:37,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:37,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:37,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-04T14:59:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-04T14:59:37,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-04T14:59:37,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0500 sec 2024-12-04T14:59:37,484 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.0590 sec 2024-12-04T14:59:37,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T14:59:37,580 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:37,581 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:59:37,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-12-04T14:59:37,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T14:59:37,587 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:59:37,587 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:59:37,587 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:59:37,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:37,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T14:59:37,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:37,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:37,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:37,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:37,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:37,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:37,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/b270e22452294d169e2d70ffcce444f0 is 50, key is test_row_0/A:col10/1733324377651/Put/seqid=0 2024-12-04T14:59:37,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742058_1234 (size=9657) 2024-12-04T14:59:37,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733324377676, max timestamp=1733324377685Num uniques 859; 2024-12-04T14:59:37,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:37,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733324377676, max timestamp=1733324377685Num uniques 859; 2024-12-04T14:59:37,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:37,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733324377676, max timestamp=1733324377685Num uniques 859; 2024-12-04T14:59:37,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T14:59:37,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:37,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324437704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324437706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:37,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324437707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:37,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324437722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:37,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:37,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324437724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:37,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:37,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:37,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:37,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-04T14:59:37,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:37,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:37,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:37,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:37,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:37,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324437830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:37,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:37,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:37,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:37,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324437830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:37,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:37,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:37,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:37,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:37,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324437830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324437830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:37,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324437830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:37,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-04T14:59:37,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:37,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:37,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:37,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:37,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T14:59:38,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324438040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:38,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324438040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:38,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:38,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324438040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324438041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:38,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324438041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:38,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-04T14:59:38,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:38,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:38,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:38,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:38,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:38,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/b270e22452294d169e2d70ffcce444f0 2024-12-04T14:59:38,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/fad1482a72594bbc82bd06e9d356587c is 50, key is test_row_0/B:col10/1733324377651/Put/seqid=0 2024-12-04T14:59:38,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742059_1235 (size=9657) 2024-12-04T14:59:38,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/fad1482a72594bbc82bd06e9d356587c 2024-12-04T14:59:38,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/db5409623fdc4f898869da96a6314101 is 50, key is test_row_0/C:col10/1733324377651/Put/seqid=0 2024-12-04T14:59:38,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742060_1236 (size=9657) 2024-12-04T14:59:38,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-04T14:59:38,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:38,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:38,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:38,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:38,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:38,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:38,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T14:59:38,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324438350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:38,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324438350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:38,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324438350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:38,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-04T14:59:38,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:38,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:38,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324438351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:38,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:38,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:38,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:38,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324438351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:38,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:38,402 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T14:59:38,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-04T14:59:38,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:38,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:38,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:38,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:38,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:38,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/db5409623fdc4f898869da96a6314101 2024-12-04T14:59:38,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/b270e22452294d169e2d70ffcce444f0 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/b270e22452294d169e2d70ffcce444f0 2024-12-04T14:59:38,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/b270e22452294d169e2d70ffcce444f0, entries=100, sequenceid=57, filesize=9.4 K 2024-12-04T14:59:38,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/fad1482a72594bbc82bd06e9d356587c as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/fad1482a72594bbc82bd06e9d356587c 2024-12-04T14:59:38,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/fad1482a72594bbc82bd06e9d356587c, entries=100, sequenceid=57, filesize=9.4 K 2024-12-04T14:59:38,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/db5409623fdc4f898869da96a6314101 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/db5409623fdc4f898869da96a6314101 2024-12-04T14:59:38,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/db5409623fdc4f898869da96a6314101, entries=100, sequenceid=57, filesize=9.4 K 2024-12-04T14:59:38,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for cf9bed3c8089ea388e8abd4912312aee in 891ms, sequenceid=57, compaction requested=true 2024-12-04T14:59:38,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:38,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:59:38,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:38,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:59:38,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:38,544 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:38,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:59:38,544 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:38,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:38,545 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:38,545 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:38,545 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/B is initiating minor compaction (all files) 2024-12-04T14:59:38,545 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/A is initiating minor compaction (all files) 2024-12-04T14:59:38,545 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/B in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:38,545 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/A in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:38,545 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/3821ef0d222647cf8e64cf8f44606b4b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/d175a81cfc8f44529c1eb682adb4e370, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/fad1482a72594bbc82bd06e9d356587c] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=32.9 K 2024-12-04T14:59:38,545 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/2e36d62ec39e45cabaeadd8efa4c1fe4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/b98c0fb81b0a4ae28d3fa5d4a401a6db, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/b270e22452294d169e2d70ffcce444f0] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=32.9 K 2024-12-04T14:59:38,545 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 3821ef0d222647cf8e64cf8f44606b4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733324375432 2024-12-04T14:59:38,545 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e36d62ec39e45cabaeadd8efa4c1fe4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733324375432 2024-12-04T14:59:38,545 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting d175a81cfc8f44529c1eb682adb4e370, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733324375489 2024-12-04T14:59:38,545 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting b98c0fb81b0a4ae28d3fa5d4a401a6db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733324375489 2024-12-04T14:59:38,546 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting fad1482a72594bbc82bd06e9d356587c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733324377649 2024-12-04T14:59:38,546 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting b270e22452294d169e2d70ffcce444f0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733324377649 2024-12-04T14:59:38,552 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#B#compaction#192 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:38,552 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#A#compaction#193 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:38,552 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/e8c3247d690c4c2891fec7db940fb2eb is 50, key is test_row_0/A:col10/1733324377651/Put/seqid=0 2024-12-04T14:59:38,552 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/7ba4de2ffaba4f3696d9dc22b470db19 is 50, key is test_row_0/B:col10/1733324377651/Put/seqid=0 2024-12-04T14:59:38,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742061_1237 (size=12104) 2024-12-04T14:59:38,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742062_1238 (size=12104) 2024-12-04T14:59:38,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-04T14:59:38,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:38,662 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-04T14:59:38,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:38,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:38,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:38,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:38,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:38,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:38,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/885f406099d141c6b22274f98d5351eb is 50, key is test_row_0/A:col10/1733324377706/Put/seqid=0 2024-12-04T14:59:38,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742063_1239 (size=12001) 2024-12-04T14:59:38,671 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/885f406099d141c6b22274f98d5351eb 2024-12-04T14:59:38,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/29516cc4fda64541aa1dd5c17b69e5cd is 50, key is test_row_0/B:col10/1733324377706/Put/seqid=0 2024-12-04T14:59:38,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742064_1240 (size=12001) 2024-12-04T14:59:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T14:59:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:38,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:38,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324438864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:38,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324438865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:38,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324438865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:38,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:38,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:38,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324438869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324438867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:38,968 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/e8c3247d690c4c2891fec7db940fb2eb as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e8c3247d690c4c2891fec7db940fb2eb 2024-12-04T14:59:38,972 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/A of cf9bed3c8089ea388e8abd4912312aee into e8c3247d690c4c2891fec7db940fb2eb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:38,972 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:38,972 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/A, priority=13, startTime=1733324378543; duration=0sec 2024-12-04T14:59:38,972 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:38,972 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:A 2024-12-04T14:59:38,972 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:38,973 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:38,973 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/C is initiating minor compaction (all files) 2024-12-04T14:59:38,973 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/C in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:38,973 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/306f1512844d4026aaf0cb8688516868, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/464c6bab6cad4169945f35d6b066d622, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/db5409623fdc4f898869da96a6314101] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=32.9 K 2024-12-04T14:59:38,973 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 306f1512844d4026aaf0cb8688516868, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733324375432 2024-12-04T14:59:38,974 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 464c6bab6cad4169945f35d6b066d622, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733324375489 2024-12-04T14:59:38,974 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting db5409623fdc4f898869da96a6314101, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733324377649 2024-12-04T14:59:38,980 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#C#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:38,981 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/0809e007e136429699aca611d3c11c90 is 50, key is test_row_0/C:col10/1733324377651/Put/seqid=0 2024-12-04T14:59:38,981 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/7ba4de2ffaba4f3696d9dc22b470db19 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/7ba4de2ffaba4f3696d9dc22b470db19 2024-12-04T14:59:38,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324438980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:38,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324438981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:38,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324438981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:38,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324438981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:38,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:38,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324438981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:38,988 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/B of cf9bed3c8089ea388e8abd4912312aee into 7ba4de2ffaba4f3696d9dc22b470db19(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:38,988 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:38,988 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/B, priority=13, startTime=1733324378544; duration=0sec 2024-12-04T14:59:38,988 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:38,988 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:B 2024-12-04T14:59:38,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742065_1241 (size=12104) 2024-12-04T14:59:39,002 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/0809e007e136429699aca611d3c11c90 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/0809e007e136429699aca611d3c11c90 2024-12-04T14:59:39,009 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/C of cf9bed3c8089ea388e8abd4912312aee into 0809e007e136429699aca611d3c11c90(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:39,009 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:39,009 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/C, priority=13, startTime=1733324378544; duration=0sec 2024-12-04T14:59:39,009 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:39,009 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:C 2024-12-04T14:59:39,083 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/29516cc4fda64541aa1dd5c17b69e5cd 2024-12-04T14:59:39,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/150463d083ba45cd990cdd35d84350a3 is 50, key is test_row_0/C:col10/1733324377706/Put/seqid=0 2024-12-04T14:59:39,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742066_1242 (size=12001) 2024-12-04T14:59:39,099 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/150463d083ba45cd990cdd35d84350a3 2024-12-04T14:59:39,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/885f406099d141c6b22274f98d5351eb as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/885f406099d141c6b22274f98d5351eb 2024-12-04T14:59:39,109 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/885f406099d141c6b22274f98d5351eb, entries=150, sequenceid=93, filesize=11.7 K 2024-12-04T14:59:39,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/29516cc4fda64541aa1dd5c17b69e5cd as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/29516cc4fda64541aa1dd5c17b69e5cd 2024-12-04T14:59:39,116 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/29516cc4fda64541aa1dd5c17b69e5cd, entries=150, sequenceid=93, filesize=11.7 K 2024-12-04T14:59:39,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/150463d083ba45cd990cdd35d84350a3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/150463d083ba45cd990cdd35d84350a3 2024-12-04T14:59:39,122 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/150463d083ba45cd990cdd35d84350a3, entries=150, sequenceid=93, filesize=11.7 K 2024-12-04T14:59:39,123 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for cf9bed3c8089ea388e8abd4912312aee in 462ms, sequenceid=93, compaction requested=false 2024-12-04T14:59:39,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:39,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:39,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-04T14:59:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-04T14:59:39,128 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-04T14:59:39,128 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5380 sec 2024-12-04T14:59:39,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.5470 sec 2024-12-04T14:59:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:39,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T14:59:39,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:39,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:39,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:39,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:39,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:39,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:39,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/f661fc9c9c724851a2060722cba61394 is 50, key is test_row_0/A:col10/1733324379190/Put/seqid=0 2024-12-04T14:59:39,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733324379209, max timestamp=1733324379219Num uniques 859; 2024-12-04T14:59:39,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:39,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733324379209, max timestamp=1733324379219Num uniques 859; 2024-12-04T14:59:39,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:39,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733324379209, max timestamp=1733324379219Num uniques 859; 2024-12-04T14:59:39,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:39,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742067_1243 (size=12001) 2024-12-04T14:59:39,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324439242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:39,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:39,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324439243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:39,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:39,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324439251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:39,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324439251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:39,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:39,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324439251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:39,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:39,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:39,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324439360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:39,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324439360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:39,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:39,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324439360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:39,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:39,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324439370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:39,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324439370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:39,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:39,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324439571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:39,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324439571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:39,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324439571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:39,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324439580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324439580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:39,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/f661fc9c9c724851a2060722cba61394 2024-12-04T14:59:39,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/df7ebbde767e420e89d12c38ea31257f is 50, key is test_row_0/B:col10/1733324379190/Put/seqid=0 2024-12-04T14:59:39,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742068_1244 (size=12001) 2024-12-04T14:59:39,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/df7ebbde767e420e89d12c38ea31257f 2024-12-04T14:59:39,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/946b04b9318f484383c706d77af2f452 is 50, key is test_row_0/C:col10/1733324379190/Put/seqid=0 2024-12-04T14:59:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T14:59:39,731 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:39,732 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:59:39,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-04T14:59:39,735 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:59:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T14:59:39,736 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:59:39,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:59:39,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742069_1245 (size=12001) 2024-12-04T14:59:39,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/946b04b9318f484383c706d77af2f452 2024-12-04T14:59:39,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/f661fc9c9c724851a2060722cba61394 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/f661fc9c9c724851a2060722cba61394 2024-12-04T14:59:39,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/f661fc9c9c724851a2060722cba61394, entries=150, sequenceid=108, filesize=11.7 K 2024-12-04T14:59:39,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/df7ebbde767e420e89d12c38ea31257f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/df7ebbde767e420e89d12c38ea31257f 2024-12-04T14:59:39,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/df7ebbde767e420e89d12c38ea31257f, entries=150, sequenceid=108, filesize=11.7 K 2024-12-04T14:59:39,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/946b04b9318f484383c706d77af2f452 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/946b04b9318f484383c706d77af2f452 2024-12-04T14:59:39,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/946b04b9318f484383c706d77af2f452, entries=150, sequenceid=108, filesize=11.7 K 2024-12-04T14:59:39,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for cf9bed3c8089ea388e8abd4912312aee in 580ms, sequenceid=108, compaction requested=true 2024-12-04T14:59:39,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:39,772 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:39,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:59:39,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:39,772 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:39,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:59:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:59:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:39,773 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:39,773 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/B is initiating minor compaction (all files) 2024-12-04T14:59:39,774 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/B in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:39,774 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/7ba4de2ffaba4f3696d9dc22b470db19, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/29516cc4fda64541aa1dd5c17b69e5cd, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/df7ebbde767e420e89d12c38ea31257f] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=35.3 K 2024-12-04T14:59:39,775 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:39,775 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/A is initiating minor compaction (all files) 2024-12-04T14:59:39,776 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/A in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:39,776 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ba4de2ffaba4f3696d9dc22b470db19, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733324375489 2024-12-04T14:59:39,776 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e8c3247d690c4c2891fec7db940fb2eb, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/885f406099d141c6b22274f98d5351eb, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/f661fc9c9c724851a2060722cba61394] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=35.3 K 2024-12-04T14:59:39,776 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 29516cc4fda64541aa1dd5c17b69e5cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733324377702 2024-12-04T14:59:39,776 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8c3247d690c4c2891fec7db940fb2eb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733324375489 2024-12-04T14:59:39,776 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting df7ebbde767e420e89d12c38ea31257f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733324378864 2024-12-04T14:59:39,776 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 885f406099d141c6b22274f98d5351eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733324377702 2024-12-04T14:59:39,779 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f661fc9c9c724851a2060722cba61394, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733324378864 2024-12-04T14:59:39,792 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#B#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:39,792 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#A#compaction#201 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:39,793 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/32c7180e1f8e4e77b27b5fee54b2bb39 is 50, key is test_row_0/A:col10/1733324379190/Put/seqid=0 2024-12-04T14:59:39,793 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/24a2c7115df9470eb475ce68177e364a is 50, key is test_row_0/B:col10/1733324379190/Put/seqid=0 2024-12-04T14:59:39,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742070_1246 (size=12207) 2024-12-04T14:59:39,818 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/32c7180e1f8e4e77b27b5fee54b2bb39 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/32c7180e1f8e4e77b27b5fee54b2bb39 2024-12-04T14:59:39,823 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/A of cf9bed3c8089ea388e8abd4912312aee into 32c7180e1f8e4e77b27b5fee54b2bb39(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:39,823 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:39,823 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/A, priority=13, startTime=1733324379772; duration=0sec 2024-12-04T14:59:39,823 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:39,823 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:A 2024-12-04T14:59:39,824 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:39,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742071_1247 (size=12207) 2024-12-04T14:59:39,825 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:39,825 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/C is initiating minor compaction (all files) 2024-12-04T14:59:39,825 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/C in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:39,825 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/0809e007e136429699aca611d3c11c90, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/150463d083ba45cd990cdd35d84350a3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/946b04b9318f484383c706d77af2f452] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=35.3 K 2024-12-04T14:59:39,826 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0809e007e136429699aca611d3c11c90, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733324375489 2024-12-04T14:59:39,827 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 150463d083ba45cd990cdd35d84350a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733324377702 2024-12-04T14:59:39,827 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 946b04b9318f484383c706d77af2f452, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733324378864 2024-12-04T14:59:39,836 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/24a2c7115df9470eb475ce68177e364a as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/24a2c7115df9470eb475ce68177e364a 2024-12-04T14:59:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T14:59:39,852 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/B of cf9bed3c8089ea388e8abd4912312aee into 24a2c7115df9470eb475ce68177e364a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:39,852 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:39,852 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/B, priority=13, startTime=1733324379772; duration=0sec 2024-12-04T14:59:39,852 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:39,852 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:B 2024-12-04T14:59:39,855 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#C#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:39,856 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/13e84ff1f3914664a042977ddcc76d9f is 50, key is test_row_0/C:col10/1733324379190/Put/seqid=0 2024-12-04T14:59:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:39,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-04T14:59:39,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:39,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:39,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:39,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:39,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:39,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:39,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-04T14:59:39,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:39,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:39,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:39,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:39,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:39,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324439893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:39,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:39,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324439894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:39,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:39,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324439896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:39,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324439896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:39,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:39,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:39,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:39,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324439901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:39,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742072_1248 (size=12207) 2024-12-04T14:59:39,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/237166f05886495b9ad0467b397db2ec is 50, key is test_row_0/A:col10/1733324379882/Put/seqid=0 2024-12-04T14:59:39,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742073_1249 (size=14541) 2024-12-04T14:59:39,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/237166f05886495b9ad0467b397db2ec 2024-12-04T14:59:39,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/ba63c803d96041279b30f15c11a6345f is 50, key is test_row_0/B:col10/1733324379882/Put/seqid=0 2024-12-04T14:59:39,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742074_1250 (size=12151) 2024-12-04T14:59:39,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/ba63c803d96041279b30f15c11a6345f 2024-12-04T14:59:40,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324440010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:40,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324440010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:40,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/efd2f68a03494af087ca10d81d67dfc6 is 50, key is test_row_0/C:col10/1733324379882/Put/seqid=0 2024-12-04T14:59:40,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324440021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:40,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742075_1251 (size=12151) 2024-12-04T14:59:40,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/efd2f68a03494af087ca10d81d67dfc6 2024-12-04T14:59:40,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/237166f05886495b9ad0467b397db2ec as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/237166f05886495b9ad0467b397db2ec 2024-12-04T14:59:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T14:59:40,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-04T14:59:40,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:40,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:40,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:40,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:40,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:40,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/237166f05886495b9ad0467b397db2ec, entries=200, sequenceid=147, filesize=14.2 K 2024-12-04T14:59:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:40,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/ba63c803d96041279b30f15c11a6345f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/ba63c803d96041279b30f15c11a6345f 2024-12-04T14:59:40,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/ba63c803d96041279b30f15c11a6345f, entries=150, sequenceid=147, filesize=11.9 K 2024-12-04T14:59:40,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/efd2f68a03494af087ca10d81d67dfc6 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/efd2f68a03494af087ca10d81d67dfc6 2024-12-04T14:59:40,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/efd2f68a03494af087ca10d81d67dfc6, entries=150, sequenceid=147, filesize=11.9 K 2024-12-04T14:59:40,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for cf9bed3c8089ea388e8abd4912312aee in 197ms, sequenceid=147, compaction requested=false 2024-12-04T14:59:40,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:40,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-04T14:59:40,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:40,209 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-04T14:59:40,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:40,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:40,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:40,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:40,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:40,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:40,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/25a9edbc18b142f5b8b8c4a4740e6cbb is 50, key is test_row_0/A:col10/1733324379894/Put/seqid=0 2024-12-04T14:59:40,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:40,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:40,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742076_1252 (size=12151) 2024-12-04T14:59:40,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324380247, max timestamp=1733324380253Num uniques 859; 2024-12-04T14:59:40,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:40,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324380247, max timestamp=1733324380253Num uniques 859; 2024-12-04T14:59:40,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:40,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324380247, max timestamp=1733324380253Num uniques 859; 2024-12-04T14:59:40,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:40,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324440280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:40,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324440281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:40,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324440285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:40,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:40,315 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/13e84ff1f3914664a042977ddcc76d9f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/13e84ff1f3914664a042977ddcc76d9f 2024-12-04T14:59:40,319 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/C of cf9bed3c8089ea388e8abd4912312aee into 13e84ff1f3914664a042977ddcc76d9f(size=11.9 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:40,319 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:40,319 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/C, priority=13, startTime=1733324379773; duration=0sec 2024-12-04T14:59:40,319 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:40,319 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:C 2024-12-04T14:59:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T14:59:40,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324440389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:40,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324440390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:40,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:40,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:40,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324440400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:40,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:40,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324440410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:40,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324440410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:40,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:40,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324440600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:40,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:40,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324440600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:40,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:40,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:40,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:40,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324440610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:40,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:40,644 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/25a9edbc18b142f5b8b8c4a4740e6cbb 2024-12-04T14:59:40,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/c920507d10ef41f3842d8ceb0ffd7573 is 50, key is test_row_0/B:col10/1733324379894/Put/seqid=0 2024-12-04T14:59:40,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742077_1253 (size=12151) 2024-12-04T14:59:40,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T14:59:40,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:40,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324440910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:40,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:40,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324440910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:40,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:40,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:40,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:40,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324440921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:40,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:40,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:41,059 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/c920507d10ef41f3842d8ceb0ffd7573 2024-12-04T14:59:41,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/340244ad88b14dae893e676157547927 is 50, key is test_row_0/C:col10/1733324379894/Put/seqid=0 2024-12-04T14:59:41,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742078_1254 (size=12151) 2024-12-04T14:59:41,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:41,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:41,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324441420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:41,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324441420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:41,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:41,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:41,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324441420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:41,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:41,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:41,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:41,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:41,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:41,423 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_1' in cf9bed3c8089ea388e8abd4912312aee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:41,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:41,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:41,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:41,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:41,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324441431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:41,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:41,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324441431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:41,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:41,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:41,434 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_1' in cf9bed3c8089ea388e8abd4912312aee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:41,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:41,472 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/340244ad88b14dae893e676157547927 2024-12-04T14:59:41,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/25a9edbc18b142f5b8b8c4a4740e6cbb as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/25a9edbc18b142f5b8b8c4a4740e6cbb 2024-12-04T14:59:41,486 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/25a9edbc18b142f5b8b8c4a4740e6cbb, entries=150, sequenceid=156, filesize=11.9 K 2024-12-04T14:59:41,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/c920507d10ef41f3842d8ceb0ffd7573 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/c920507d10ef41f3842d8ceb0ffd7573 2024-12-04T14:59:41,491 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/c920507d10ef41f3842d8ceb0ffd7573, entries=150, sequenceid=156, filesize=11.9 K 2024-12-04T14:59:41,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/340244ad88b14dae893e676157547927 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/340244ad88b14dae893e676157547927 2024-12-04T14:59:41,497 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/340244ad88b14dae893e676157547927, entries=150, sequenceid=156, filesize=11.9 K 2024-12-04T14:59:41,499 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for cf9bed3c8089ea388e8abd4912312aee in 1290ms, sequenceid=156, compaction requested=true 2024-12-04T14:59:41,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:41,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:41,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-04T14:59:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-04T14:59:41,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-04T14:59:41,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7660 sec 2024-12-04T14:59:41,509 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.7750 sec 2024-12-04T14:59:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T14:59:41,880 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:41,880 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:59:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-04T14:59:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T14:59:41,882 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:59:41,883 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:59:41,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:59:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T14:59:42,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T14:59:42,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:42,037 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-04T14:59:42,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:42,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:42,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:42,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:42,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:42,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:42,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/595b6be5603f4c118af529631d74a897 is 50, key is test_row_0/A:col10/1733324380284/Put/seqid=0 2024-12-04T14:59:42,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742079_1255 (size=12151) 2024-12-04T14:59:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T14:59:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:42,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:42,445 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/595b6be5603f4c118af529631d74a897 2024-12-04T14:59:42,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324442444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:42,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324442447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:42,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:42,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:42,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:42,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324442450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:42,453 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_1' in cf9bed3c8089ea388e8abd4912312aee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:42,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/16b281418e2348d7bbe0af8d188e04d0 is 50, key is test_row_0/B:col10/1733324380284/Put/seqid=0 2024-12-04T14:59:42,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742080_1256 (size=12151) 2024-12-04T14:59:42,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T14:59:42,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324442560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:42,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324442560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:42,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324442770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:42,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:42,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324442770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:42,859 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/16b281418e2348d7bbe0af8d188e04d0 2024-12-04T14:59:42,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/f50464766cc549ebba761e92ba618d09 is 50, key is test_row_0/C:col10/1733324380284/Put/seqid=0 2024-12-04T14:59:42,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742081_1257 (size=12151) 2024-12-04T14:59:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T14:59:43,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:43,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324443080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:43,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:43,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324443080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:43,281 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/f50464766cc549ebba761e92ba618d09 2024-12-04T14:59:43,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/595b6be5603f4c118af529631d74a897 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/595b6be5603f4c118af529631d74a897 2024-12-04T14:59:43,298 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/595b6be5603f4c118af529631d74a897, entries=150, sequenceid=194, filesize=11.9 K 2024-12-04T14:59:43,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/16b281418e2348d7bbe0af8d188e04d0 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/16b281418e2348d7bbe0af8d188e04d0 2024-12-04T14:59:43,303 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/16b281418e2348d7bbe0af8d188e04d0, entries=150, sequenceid=194, filesize=11.9 K 2024-12-04T14:59:43,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/f50464766cc549ebba761e92ba618d09 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/f50464766cc549ebba761e92ba618d09 2024-12-04T14:59:43,310 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/f50464766cc549ebba761e92ba618d09, entries=150, sequenceid=194, filesize=11.9 K 2024-12-04T14:59:43,313 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for cf9bed3c8089ea388e8abd4912312aee in 1275ms, sequenceid=194, compaction requested=true 2024-12-04T14:59:43,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:43,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:43,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-04T14:59:43,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-04T14:59:43,316 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-04T14:59:43,316 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4310 sec 2024-12-04T14:59:43,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.4370 sec 2024-12-04T14:59:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:43,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T14:59:43,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:43,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:43,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:43,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:43,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:43,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:43,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/ca4148a1672f453ebb9ae0630eb40226 is 50, key is test_row_0/A:col10/1733324382445/Put/seqid=0 2024-12-04T14:59:43,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742082_1258 (size=12151) 2024-12-04T14:59:43,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/ca4148a1672f453ebb9ae0630eb40226 2024-12-04T14:59:43,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/c1450696ef0f4716b30bb992c2a16f37 is 50, key is test_row_0/B:col10/1733324382445/Put/seqid=0 2024-12-04T14:59:43,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742083_1259 (size=12151) 2024-12-04T14:59:43,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324383490, max timestamp=1733324383497Num uniques 859; 2024-12-04T14:59:43,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:59:43,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324383490, max timestamp=1733324383497Num uniques 859; 2024-12-04T14:59:43,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:59:43,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324383490, max timestamp=1733324383497Num uniques 859; 2024-12-04T14:59:43,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:59:43,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:43,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324443538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:43,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324443539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:43,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:43,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:43,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:43,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324443590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:43,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324443590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:43,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324443650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:43,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324443650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:43,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324443860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:43,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:43,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:43,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324443860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:43,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:43,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:43,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:43,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:43,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/c1450696ef0f4716b30bb992c2a16f37 2024-12-04T14:59:43,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/e794136acdc448bf96864e631563675d is 50, key is test_row_0/C:col10/1733324382445/Put/seqid=0 2024-12-04T14:59:43,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742084_1260 (size=12151) 2024-12-04T14:59:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T14:59:44,030 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:44,031 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:59:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-04T14:59:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T14:59:44,032 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:59:44,033 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:59:44,033 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:59:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T14:59:44,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:44,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324444169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:44,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:44,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:44,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:44,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324444170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:44,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:44,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:44,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:44,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:44,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-04T14:59:44,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:44,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:44,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:44,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:44,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:44,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/e794136acdc448bf96864e631563675d 2024-12-04T14:59:44,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/ca4148a1672f453ebb9ae0630eb40226 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/ca4148a1672f453ebb9ae0630eb40226 2024-12-04T14:59:44,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/ca4148a1672f453ebb9ae0630eb40226, entries=150, sequenceid=204, filesize=11.9 K 2024-12-04T14:59:44,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/c1450696ef0f4716b30bb992c2a16f37 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/c1450696ef0f4716b30bb992c2a16f37 2024-12-04T14:59:44,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/c1450696ef0f4716b30bb992c2a16f37, entries=150, sequenceid=204, filesize=11.9 K 2024-12-04T14:59:44,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/e794136acdc448bf96864e631563675d as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/e794136acdc448bf96864e631563675d 2024-12-04T14:59:44,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/e794136acdc448bf96864e631563675d, entries=150, sequenceid=204, filesize=11.9 K 2024-12-04T14:59:44,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for cf9bed3c8089ea388e8abd4912312aee in 880ms, sequenceid=204, compaction requested=true 2024-12-04T14:59:44,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:44,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:59:44,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:44,324 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T14:59:44,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:59:44,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:44,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:59:44,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:44,324 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T14:59:44,325 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60811 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T14:59:44,325 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 63201 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T14:59:44,325 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/B is initiating minor compaction (all files) 2024-12-04T14:59:44,325 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/A is initiating minor compaction (all files) 2024-12-04T14:59:44,325 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/B in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:44,325 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/A in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:44,325 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/32c7180e1f8e4e77b27b5fee54b2bb39, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/237166f05886495b9ad0467b397db2ec, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/25a9edbc18b142f5b8b8c4a4740e6cbb, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/595b6be5603f4c118af529631d74a897, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/ca4148a1672f453ebb9ae0630eb40226] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=61.7 K 2024-12-04T14:59:44,325 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/24a2c7115df9470eb475ce68177e364a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/ba63c803d96041279b30f15c11a6345f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/c920507d10ef41f3842d8ceb0ffd7573, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/16b281418e2348d7bbe0af8d188e04d0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/c1450696ef0f4716b30bb992c2a16f37] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=59.4 K 2024-12-04T14:59:44,325 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 24a2c7115df9470eb475ce68177e364a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733324378864 2024-12-04T14:59:44,325 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32c7180e1f8e4e77b27b5fee54b2bb39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733324378864 2024-12-04T14:59:44,326 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting ba63c803d96041279b30f15c11a6345f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733324379244 2024-12-04T14:59:44,326 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 237166f05886495b9ad0467b397db2ec, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733324379244 2024-12-04T14:59:44,326 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting c920507d10ef41f3842d8ceb0ffd7573, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733324379888 2024-12-04T14:59:44,326 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25a9edbc18b142f5b8b8c4a4740e6cbb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733324379888 2024-12-04T14:59:44,326 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 16b281418e2348d7bbe0af8d188e04d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733324380272 2024-12-04T14:59:44,326 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 595b6be5603f4c118af529631d74a897, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733324380272 2024-12-04T14:59:44,326 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting c1450696ef0f4716b30bb992c2a16f37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733324382432 2024-12-04T14:59:44,326 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca4148a1672f453ebb9ae0630eb40226, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733324382432 2024-12-04T14:59:44,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-04T14:59:44,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:44,341 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-04T14:59:44,341 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#B#compaction#216 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:44,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:44,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:44,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:44,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:44,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:44,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:44,341 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/d35f685c6ea14fe6bf32493616032ffb is 50, key is test_row_0/B:col10/1733324382445/Put/seqid=0 2024-12-04T14:59:44,348 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#A#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:44,349 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/3398e1f6267f46b5a6103bc41bdbb816 is 50, key is test_row_0/A:col10/1733324382445/Put/seqid=0 2024-12-04T14:59:44,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/12c5df6a6aaf42deb5755fa1ed42ac52 is 50, key is test_row_0/A:col10/1733324383533/Put/seqid=0 2024-12-04T14:59:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T14:59:44,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742085_1261 (size=12527) 2024-12-04T14:59:44,364 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/d35f685c6ea14fe6bf32493616032ffb as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/d35f685c6ea14fe6bf32493616032ffb 2024-12-04T14:59:44,369 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/B of cf9bed3c8089ea388e8abd4912312aee into d35f685c6ea14fe6bf32493616032ffb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:44,369 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:44,369 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/B, priority=11, startTime=1733324384324; duration=0sec 2024-12-04T14:59:44,369 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:44,369 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:B 2024-12-04T14:59:44,369 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T14:59:44,372 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60811 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T14:59:44,372 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/C is initiating minor compaction (all files) 2024-12-04T14:59:44,372 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/C in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:44,372 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/13e84ff1f3914664a042977ddcc76d9f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/efd2f68a03494af087ca10d81d67dfc6, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/340244ad88b14dae893e676157547927, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/f50464766cc549ebba761e92ba618d09, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/e794136acdc448bf96864e631563675d] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=59.4 K 2024-12-04T14:59:44,373 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 13e84ff1f3914664a042977ddcc76d9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733324378864 2024-12-04T14:59:44,373 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting efd2f68a03494af087ca10d81d67dfc6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733324379244 2024-12-04T14:59:44,373 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 340244ad88b14dae893e676157547927, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733324379888 2024-12-04T14:59:44,373 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting f50464766cc549ebba761e92ba618d09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733324380272 2024-12-04T14:59:44,374 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting e794136acdc448bf96864e631563675d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733324382432 2024-12-04T14:59:44,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742086_1262 (size=12527) 2024-12-04T14:59:44,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742087_1263 (size=12151) 2024-12-04T14:59:44,386 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#C#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:44,386 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/a1665cbfac0f4bc7abc35c3c59370330 is 50, key is test_row_0/C:col10/1733324382445/Put/seqid=0 2024-12-04T14:59:44,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742088_1264 (size=12527) 2024-12-04T14:59:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:44,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:44,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324444502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:44,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:44,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:44,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:44,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324444600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:44,602 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_0' in cf9bed3c8089ea388e8abd4912312aee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:44,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:44,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324444610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:44,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324444610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:44,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:44,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:44,612 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_0' in cf9bed3c8089ea388e8abd4912312aee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:44,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:44,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:44,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:44,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T14:59:44,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324444680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:44,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:44,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324444680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:44,784 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/12c5df6a6aaf42deb5755fa1ed42ac52 2024-12-04T14:59:44,788 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/3398e1f6267f46b5a6103bc41bdbb816 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/3398e1f6267f46b5a6103bc41bdbb816 2024-12-04T14:59:44,793 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/A of cf9bed3c8089ea388e8abd4912312aee into 3398e1f6267f46b5a6103bc41bdbb816(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:44,793 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:44,793 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/A, priority=11, startTime=1733324384323; duration=0sec 2024-12-04T14:59:44,793 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:44,793 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:A 2024-12-04T14:59:44,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/0926ed29787f463dbf19a6036708c134 is 50, key is test_row_0/B:col10/1733324383533/Put/seqid=0 2024-12-04T14:59:44,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742089_1265 (size=12151) 2024-12-04T14:59:44,812 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/a1665cbfac0f4bc7abc35c3c59370330 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/a1665cbfac0f4bc7abc35c3c59370330 2024-12-04T14:59:44,817 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/C of cf9bed3c8089ea388e8abd4912312aee into a1665cbfac0f4bc7abc35c3c59370330(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:44,817 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:44,817 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/C, priority=11, startTime=1733324384324; duration=0sec 2024-12-04T14:59:44,817 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:44,817 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:C 2024-12-04T14:59:44,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:44,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324444820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:45,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:45,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324445130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:45,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:45,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:45,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T14:59:45,204 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/0926ed29787f463dbf19a6036708c134 2024-12-04T14:59:45,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/2cd3b45b1d084c04b999f19f38e66307 is 50, key is test_row_0/C:col10/1733324383533/Put/seqid=0 2024-12-04T14:59:45,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742090_1266 (size=12151) 2024-12-04T14:59:45,614 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/2cd3b45b1d084c04b999f19f38e66307 2024-12-04T14:59:45,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/12c5df6a6aaf42deb5755fa1ed42ac52 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/12c5df6a6aaf42deb5755fa1ed42ac52 2024-12-04T14:59:45,622 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/12c5df6a6aaf42deb5755fa1ed42ac52, entries=150, sequenceid=241, filesize=11.9 K 2024-12-04T14:59:45,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/0926ed29787f463dbf19a6036708c134 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/0926ed29787f463dbf19a6036708c134 2024-12-04T14:59:45,626 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/0926ed29787f463dbf19a6036708c134, entries=150, sequenceid=241, filesize=11.9 K 2024-12-04T14:59:45,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/2cd3b45b1d084c04b999f19f38e66307 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/2cd3b45b1d084c04b999f19f38e66307 2024-12-04T14:59:45,631 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/2cd3b45b1d084c04b999f19f38e66307, entries=150, sequenceid=241, filesize=11.9 K 2024-12-04T14:59:45,632 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for cf9bed3c8089ea388e8abd4912312aee in 1292ms, sequenceid=241, compaction requested=false 2024-12-04T14:59:45,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:45,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:45,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-04T14:59:45,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-04T14:59:45,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-04T14:59:45,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6010 sec 2024-12-04T14:59:45,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.6050 sec 2024-12-04T14:59:45,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:45,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T14:59:45,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:45,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:45,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:45,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:45,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:45,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:45,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/baf4660076e2411a8376234d408feb33 is 50, key is test_row_0/A:col10/1733324385641/Put/seqid=0 2024-12-04T14:59:45,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742091_1267 (size=12151) 2024-12-04T14:59:45,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324385684, max timestamp=1733324385695Num uniques 859; 2024-12-04T14:59:45,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:45,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324385684, max timestamp=1733324385695Num uniques 859; 2024-12-04T14:59:45,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:45,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324385684, max timestamp=1733324385695Num uniques 859; 2024-12-04T14:59:45,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T14:59:45,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324445716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:45,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:45,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:45,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:45,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324445721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:45,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324445721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:45,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:45,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:45,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:45,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324445830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:45,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:45,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:45,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:45,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:45,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324445830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:45,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:45,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:45,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324445830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:45,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:45,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:45,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:45,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:45,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:46,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324446040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:46,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324446040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324446040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:46,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/baf4660076e2411a8376234d408feb33 2024-12-04T14:59:46,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/27650e44c53a444f95c678490479be24 is 50, key is test_row_0/B:col10/1733324385641/Put/seqid=0 2024-12-04T14:59:46,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742092_1268 (size=12151) 2024-12-04T14:59:46,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/27650e44c53a444f95c678490479be24 2024-12-04T14:59:46,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/a298f9e38e964c1398fdda4ad31dad98 is 50, key is test_row_0/C:col10/1733324385641/Put/seqid=0 2024-12-04T14:59:46,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742093_1269 (size=12151) 2024-12-04T14:59:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T14:59:46,180 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:46,181 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:59:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-04T14:59:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T14:59:46,182 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:59:46,183 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:59:46,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:59:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T14:59:46,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T14:59:46,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:46,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:46,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:46,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:46,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:46,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:46,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324446350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:46,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324446350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:46,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:46,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324446351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:46,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/a298f9e38e964c1398fdda4ad31dad98 2024-12-04T14:59:46,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/baf4660076e2411a8376234d408feb33 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/baf4660076e2411a8376234d408feb33 2024-12-04T14:59:46,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/baf4660076e2411a8376234d408feb33, entries=150, sequenceid=255, filesize=11.9 K 2024-12-04T14:59:46,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/27650e44c53a444f95c678490479be24 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/27650e44c53a444f95c678490479be24 2024-12-04T14:59:46,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T14:59:46,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/27650e44c53a444f95c678490479be24, entries=150, sequenceid=255, filesize=11.9 K 2024-12-04T14:59:46,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:46,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:46,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:46,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:46,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:46,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/a298f9e38e964c1398fdda4ad31dad98 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/a298f9e38e964c1398fdda4ad31dad98 2024-12-04T14:59:46,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/a298f9e38e964c1398fdda4ad31dad98, entries=150, sequenceid=255, filesize=11.9 K 2024-12-04T14:59:46,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cf9bed3c8089ea388e8abd4912312aee in 852ms, sequenceid=255, compaction requested=true 2024-12-04T14:59:46,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:46,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:59:46,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:46,494 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:46,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:59:46,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:46,494 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:46,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:59:46,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:46,495 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:46,495 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:46,495 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/A is initiating minor compaction (all files) 2024-12-04T14:59:46,495 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/B is initiating minor compaction (all files) 2024-12-04T14:59:46,495 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/B in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:46,495 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/A in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:46,495 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/3398e1f6267f46b5a6103bc41bdbb816, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/12c5df6a6aaf42deb5755fa1ed42ac52, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/baf4660076e2411a8376234d408feb33] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=36.0 K 2024-12-04T14:59:46,495 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/d35f685c6ea14fe6bf32493616032ffb, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/0926ed29787f463dbf19a6036708c134, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/27650e44c53a444f95c678490479be24] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=36.0 K 2024-12-04T14:59:46,495 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting d35f685c6ea14fe6bf32493616032ffb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733324382432 2024-12-04T14:59:46,495 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3398e1f6267f46b5a6103bc41bdbb816, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733324382432 2024-12-04T14:59:46,496 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12c5df6a6aaf42deb5755fa1ed42ac52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733324383533 2024-12-04T14:59:46,496 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 0926ed29787f463dbf19a6036708c134, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733324383533 2024-12-04T14:59:46,496 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting baf4660076e2411a8376234d408feb33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733324384495 2024-12-04T14:59:46,496 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 27650e44c53a444f95c678490479be24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733324384495 2024-12-04T14:59:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T14:59:46,505 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#A#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:46,505 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#B#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:46,505 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/643d77e75a794a18ae24ca8e1dded3f9 is 50, key is test_row_0/B:col10/1733324385641/Put/seqid=0 2024-12-04T14:59:46,505 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/b8c52dfff60e47a3affd5b48933b512d is 50, key is test_row_0/A:col10/1733324385641/Put/seqid=0 2024-12-04T14:59:46,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742095_1271 (size=12629) 2024-12-04T14:59:46,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742094_1270 (size=12629) 2024-12-04T14:59:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:46,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-04T14:59:46,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:46,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:46,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:46,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:46,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:46,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:46,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/28807ab9f48a4ab8a5ffcad216523483 is 50, key is test_row_0/A:col10/1733324386631/Put/seqid=0 2024-12-04T14:59:46,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T14:59:46,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:46,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:46,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:46,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742096_1272 (size=14741) 2024-12-04T14:59:46,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:46,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:46,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324446649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:46,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:46,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:46,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324446652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:46,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:46,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324446760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:46,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:46,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324446760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:46,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:46,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:46,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T14:59:46,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:46,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:46,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:46,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:46,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:46,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T14:59:46,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:46,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324446860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:46,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:46,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:46,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324446860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:46,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:46,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324446860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:46,930 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/b8c52dfff60e47a3affd5b48933b512d as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/b8c52dfff60e47a3affd5b48933b512d 2024-12-04T14:59:46,932 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/643d77e75a794a18ae24ca8e1dded3f9 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/643d77e75a794a18ae24ca8e1dded3f9 2024-12-04T14:59:46,934 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/A of cf9bed3c8089ea388e8abd4912312aee into b8c52dfff60e47a3affd5b48933b512d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:46,934 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:46,935 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/A, priority=13, startTime=1733324386494; duration=0sec 2024-12-04T14:59:46,935 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:46,935 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:A 2024-12-04T14:59:46,935 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:46,936 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/B of cf9bed3c8089ea388e8abd4912312aee into 643d77e75a794a18ae24ca8e1dded3f9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:46,936 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:46,936 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/B, priority=13, startTime=1733324386494; duration=0sec 2024-12-04T14:59:46,936 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:46,936 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/C is initiating minor compaction (all files) 2024-12-04T14:59:46,936 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:46,936 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:B 2024-12-04T14:59:46,936 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/C in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:46,936 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/a1665cbfac0f4bc7abc35c3c59370330, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/2cd3b45b1d084c04b999f19f38e66307, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/a298f9e38e964c1398fdda4ad31dad98] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=36.0 K 2024-12-04T14:59:46,937 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1665cbfac0f4bc7abc35c3c59370330, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733324382432 2024-12-04T14:59:46,937 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cd3b45b1d084c04b999f19f38e66307, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733324383533 2024-12-04T14:59:46,941 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting a298f9e38e964c1398fdda4ad31dad98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733324384495 2024-12-04T14:59:46,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T14:59:46,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:46,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:46,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:46,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:46,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:46,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:46,951 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#C#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:46,952 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/432fd6483b6b47d19d67e15f096a3030 is 50, key is test_row_0/C:col10/1733324385641/Put/seqid=0 2024-12-04T14:59:46,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742097_1273 (size=12629) 2024-12-04T14:59:46,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324446970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:46,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324446970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:47,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/28807ab9f48a4ab8a5ffcad216523483 2024-12-04T14:59:47,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/5794a80b6a1a4605a33e554d2e1d9e68 is 50, key is test_row_0/B:col10/1733324386631/Put/seqid=0 2024-12-04T14:59:47,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742098_1274 (size=12301) 2024-12-04T14:59:47,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T14:59:47,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:47,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:47,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:47,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:47,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:47,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:47,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T14:59:47,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:47,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:47,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:47,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:47,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:47,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:47,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:47,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324447280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:47,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:47,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:47,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:47,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324447280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:47,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:47,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:47,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:47,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:47,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T14:59:47,364 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/432fd6483b6b47d19d67e15f096a3030 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/432fd6483b6b47d19d67e15f096a3030 2024-12-04T14:59:47,368 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/C of cf9bed3c8089ea388e8abd4912312aee into 432fd6483b6b47d19d67e15f096a3030(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:47,368 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:47,368 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/C, priority=13, startTime=1733324386494; duration=0sec 2024-12-04T14:59:47,368 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:47,368 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:C 2024-12-04T14:59:47,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T14:59:47,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:47,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:47,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:47,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:47,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:47,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/5794a80b6a1a4605a33e554d2e1d9e68 2024-12-04T14:59:47,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/d79e05ca0fbe467b8ef17bab8387dbb3 is 50, key is test_row_0/C:col10/1733324386631/Put/seqid=0 2024-12-04T14:59:47,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742099_1275 (size=12301) 2024-12-04T14:59:47,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/d79e05ca0fbe467b8ef17bab8387dbb3 2024-12-04T14:59:47,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/28807ab9f48a4ab8a5ffcad216523483 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/28807ab9f48a4ab8a5ffcad216523483 2024-12-04T14:59:47,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/28807ab9f48a4ab8a5ffcad216523483, entries=200, sequenceid=291, filesize=14.4 K 2024-12-04T14:59:47,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/5794a80b6a1a4605a33e554d2e1d9e68 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/5794a80b6a1a4605a33e554d2e1d9e68 2024-12-04T14:59:47,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/5794a80b6a1a4605a33e554d2e1d9e68, entries=150, sequenceid=291, filesize=12.0 K 2024-12-04T14:59:47,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/d79e05ca0fbe467b8ef17bab8387dbb3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/d79e05ca0fbe467b8ef17bab8387dbb3 2024-12-04T14:59:47,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/d79e05ca0fbe467b8ef17bab8387dbb3, entries=150, sequenceid=291, filesize=12.0 K 2024-12-04T14:59:47,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for cf9bed3c8089ea388e8abd4912312aee in 852ms, sequenceid=291, compaction requested=false 2024-12-04T14:59:47,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:47,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T14:59:47,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:47,556 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T14:59:47,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:47,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:47,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:47,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:47,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:47,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:47,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/8f4d64dc856a44a18d6c5af6845db1d4 is 50, key is test_row_0/A:col10/1733324386641/Put/seqid=0 2024-12-04T14:59:47,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742100_1276 (size=12301) 2024-12-04T14:59:47,569 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/8f4d64dc856a44a18d6c5af6845db1d4 2024-12-04T14:59:47,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/3eed1159ac7243ea8ff7718c28d0084c is 50, key is test_row_0/B:col10/1733324386641/Put/seqid=0 2024-12-04T14:59:47,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742101_1277 (size=12301) 2024-12-04T14:59:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:47,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:47,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324387822, max timestamp=1733324387831Num uniques 859; 2024-12-04T14:59:47,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:59:47,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324387822, max timestamp=1733324387831Num uniques 859; 2024-12-04T14:59:47,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:59:47,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324387822, max timestamp=1733324387831Num uniques 859; 2024-12-04T14:59:47,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:59:47,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324447861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:47,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324447865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:47,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324447880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:47,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324447880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:47,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324447881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:47,883 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in cf9bed3c8089ea388e8abd4912312aee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:47,883 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in cf9bed3c8089ea388e8abd4912312aee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:47,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:47,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:47,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:47,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:47,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:47,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:47,883 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in cf9bed3c8089ea388e8abd4912312aee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:47,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:47,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:47,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:47,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324447970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:47,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:47,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324447970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:47,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:47,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:47,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:47,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:47,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:47,987 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/3eed1159ac7243ea8ff7718c28d0084c 2024-12-04T14:59:47,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/7e15e6e91c4b46eba71aafdddb38e305 is 50, key is test_row_0/C:col10/1733324386641/Put/seqid=0 2024-12-04T14:59:47,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742102_1278 (size=12301) 2024-12-04T14:59:47,998 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/7e15e6e91c4b46eba71aafdddb38e305 2024-12-04T14:59:48,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/8f4d64dc856a44a18d6c5af6845db1d4 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/8f4d64dc856a44a18d6c5af6845db1d4 2024-12-04T14:59:48,016 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/8f4d64dc856a44a18d6c5af6845db1d4, entries=150, sequenceid=304, filesize=12.0 K 2024-12-04T14:59:48,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/3eed1159ac7243ea8ff7718c28d0084c as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/3eed1159ac7243ea8ff7718c28d0084c 2024-12-04T14:59:48,024 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/3eed1159ac7243ea8ff7718c28d0084c, entries=150, sequenceid=304, filesize=12.0 K 2024-12-04T14:59:48,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/7e15e6e91c4b46eba71aafdddb38e305 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/7e15e6e91c4b46eba71aafdddb38e305 2024-12-04T14:59:48,031 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/7e15e6e91c4b46eba71aafdddb38e305, entries=150, sequenceid=304, filesize=12.0 K 2024-12-04T14:59:48,032 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for cf9bed3c8089ea388e8abd4912312aee in 476ms, sequenceid=304, compaction requested=true 2024-12-04T14:59:48,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:48,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:48,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-04T14:59:48,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-04T14:59:48,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-04T14:59:48,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8520 sec 2024-12-04T14:59:48,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.8580 sec 2024-12-04T14:59:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:48,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-04T14:59:48,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:48,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T14:59:48,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:48,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T14:59:48,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:48,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T14:59:48,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/d183fdf678254d73a1b0151114b33af3 is 50, key is test_row_0/A:col10/1733324388182/Put/seqid=0 2024-12-04T14:59:48,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324448205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:48,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324448206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:48,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742103_1279 (size=12301) 2024-12-04T14:59:48,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/d183fdf678254d73a1b0151114b33af3 2024-12-04T14:59:48,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/0e62acb5ccff4fd99ee62a9b5e88035c is 50, key is test_row_0/B:col10/1733324388182/Put/seqid=0 2024-12-04T14:59:48,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742104_1280 (size=12301) 2024-12-04T14:59:48,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/0e62acb5ccff4fd99ee62a9b5e88035c 2024-12-04T14:59:48,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/6b9483e57a214568a06e5dd340a2284b is 50, key is test_row_0/C:col10/1733324388182/Put/seqid=0 2024-12-04T14:59:48,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742105_1281 (size=12301) 2024-12-04T14:59:48,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/6b9483e57a214568a06e5dd340a2284b 2024-12-04T14:59:48,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/d183fdf678254d73a1b0151114b33af3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/d183fdf678254d73a1b0151114b33af3 2024-12-04T14:59:48,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/d183fdf678254d73a1b0151114b33af3, entries=150, sequenceid=342, filesize=12.0 K 2024-12-04T14:59:48,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/0e62acb5ccff4fd99ee62a9b5e88035c as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/0e62acb5ccff4fd99ee62a9b5e88035c 2024-12-04T14:59:48,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/0e62acb5ccff4fd99ee62a9b5e88035c, entries=150, sequenceid=342, filesize=12.0 K 2024-12-04T14:59:48,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/6b9483e57a214568a06e5dd340a2284b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/6b9483e57a214568a06e5dd340a2284b 2024-12-04T14:59:48,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/6b9483e57a214568a06e5dd340a2284b, entries=150, sequenceid=342, filesize=12.0 K 2024-12-04T14:59:48,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for cf9bed3c8089ea388e8abd4912312aee in 140ms, sequenceid=342, compaction requested=true 2024-12-04T14:59:48,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:48,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:59:48,324 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:59:48,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:48,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:59:48,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:48,324 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:59:48,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:59:48,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:48,326 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51972 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:59:48,326 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/A is initiating minor compaction (all files) 2024-12-04T14:59:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:48,326 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/A in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:48,327 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/b8c52dfff60e47a3affd5b48933b512d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/28807ab9f48a4ab8a5ffcad216523483, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/8f4d64dc856a44a18d6c5af6845db1d4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/d183fdf678254d73a1b0151114b33af3] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=50.8 K 2024-12-04T14:59:48,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T14:59:48,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:48,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:48,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:48,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:48,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:48,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:48,328 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8c52dfff60e47a3affd5b48933b512d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733324384495 2024-12-04T14:59:48,328 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:59:48,328 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/B is initiating minor compaction (all files) 2024-12-04T14:59:48,328 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/B in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:48,328 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/643d77e75a794a18ae24ca8e1dded3f9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/5794a80b6a1a4605a33e554d2e1d9e68, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/3eed1159ac7243ea8ff7718c28d0084c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/0e62acb5ccff4fd99ee62a9b5e88035c] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=48.4 K 2024-12-04T14:59:48,328 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 643d77e75a794a18ae24ca8e1dded3f9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733324384495 2024-12-04T14:59:48,328 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28807ab9f48a4ab8a5ffcad216523483, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733324385715 2024-12-04T14:59:48,329 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 5794a80b6a1a4605a33e554d2e1d9e68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733324385715 2024-12-04T14:59:48,329 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f4d64dc856a44a18d6c5af6845db1d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733324386641 2024-12-04T14:59:48,329 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eed1159ac7243ea8ff7718c28d0084c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733324386641 2024-12-04T14:59:48,330 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting d183fdf678254d73a1b0151114b33af3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733324387854 2024-12-04T14:59:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T14:59:48,331 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e62acb5ccff4fd99ee62a9b5e88035c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733324387854 2024-12-04T14:59:48,331 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:48,331 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:59:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-04T14:59:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T14:59:48,334 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:59:48,336 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:59:48,336 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:59:48,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/e408a276f6054d64bfe113f1e47d514f is 50, key is test_row_0/A:col10/1733324388199/Put/seqid=0 2024-12-04T14:59:48,347 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#B#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:48,348 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/1d8aac34ea2c4afaaa93f522cb806a4c is 50, key is test_row_0/B:col10/1733324388182/Put/seqid=0 2024-12-04T14:59:48,350 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#A#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:48,350 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/2fb3843171d744398b3aff4b272a522c is 50, key is test_row_0/A:col10/1733324388182/Put/seqid=0 2024-12-04T14:59:48,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742107_1283 (size=12915) 2024-12-04T14:59:48,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742108_1284 (size=12915) 2024-12-04T14:59:48,372 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/1d8aac34ea2c4afaaa93f522cb806a4c as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/1d8aac34ea2c4afaaa93f522cb806a4c 2024-12-04T14:59:48,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742106_1282 (size=12301) 2024-12-04T14:59:48,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/e408a276f6054d64bfe113f1e47d514f 2024-12-04T14:59:48,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/39a1df2348464d34a9cacf3e094d2e72 is 50, key is test_row_0/B:col10/1733324388199/Put/seqid=0 2024-12-04T14:59:48,388 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/B of cf9bed3c8089ea388e8abd4912312aee into 1d8aac34ea2c4afaaa93f522cb806a4c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:48,388 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:48,388 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/B, priority=12, startTime=1733324388324; duration=0sec 2024-12-04T14:59:48,388 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:48,388 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:B 2024-12-04T14:59:48,388 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:59:48,389 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:59:48,390 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/C is initiating minor compaction (all files) 2024-12-04T14:59:48,390 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/C in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:48,390 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/432fd6483b6b47d19d67e15f096a3030, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/d79e05ca0fbe467b8ef17bab8387dbb3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/7e15e6e91c4b46eba71aafdddb38e305, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/6b9483e57a214568a06e5dd340a2284b] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=48.4 K 2024-12-04T14:59:48,390 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 432fd6483b6b47d19d67e15f096a3030, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733324384495 2024-12-04T14:59:48,391 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting d79e05ca0fbe467b8ef17bab8387dbb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733324385715 2024-12-04T14:59:48,391 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e15e6e91c4b46eba71aafdddb38e305, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733324386641 2024-12-04T14:59:48,391 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b9483e57a214568a06e5dd340a2284b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733324387854 2024-12-04T14:59:48,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742109_1285 (size=12301) 2024-12-04T14:59:48,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/39a1df2348464d34a9cacf3e094d2e72 2024-12-04T14:59:48,401 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#C#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:48,402 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/b20f8a7e58ff4808a3ba24583d96fdd6 is 50, key is test_row_0/C:col10/1733324388182/Put/seqid=0 2024-12-04T14:59:48,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742110_1286 (size=12915) 2024-12-04T14:59:48,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/aad2883f41974144adb303ee279587b2 is 50, key is test_row_0/C:col10/1733324388199/Put/seqid=0 2024-12-04T14:59:48,418 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/b20f8a7e58ff4808a3ba24583d96fdd6 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/b20f8a7e58ff4808a3ba24583d96fdd6 2024-12-04T14:59:48,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742111_1287 (size=12301) 2024-12-04T14:59:48,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/aad2883f41974144adb303ee279587b2 2024-12-04T14:59:48,423 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/C of cf9bed3c8089ea388e8abd4912312aee into b20f8a7e58ff4808a3ba24583d96fdd6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:48,423 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:48,423 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/C, priority=12, startTime=1733324388324; duration=0sec 2024-12-04T14:59:48,423 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:48,423 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:C 2024-12-04T14:59:48,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324388410, max timestamp=1733324388422Num uniques 859; 2024-12-04T14:59:48,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:59:48,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324388410, max timestamp=1733324388422Num uniques 859; 2024-12-04T14:59:48,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:59:48,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324388410, max timestamp=1733324388422Num uniques 859; 2024-12-04T14:59:48,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:59:48,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/e408a276f6054d64bfe113f1e47d514f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e408a276f6054d64bfe113f1e47d514f 2024-12-04T14:59:48,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e408a276f6054d64bfe113f1e47d514f, entries=150, sequenceid=353, filesize=12.0 K 2024-12-04T14:59:48,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/39a1df2348464d34a9cacf3e094d2e72 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/39a1df2348464d34a9cacf3e094d2e72 2024-12-04T14:59:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T14:59:48,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/39a1df2348464d34a9cacf3e094d2e72, entries=150, sequenceid=353, filesize=12.0 K 2024-12-04T14:59:48,444 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-04T14:59:48,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/aad2883f41974144adb303ee279587b2 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/aad2883f41974144adb303ee279587b2 2024-12-04T14:59:48,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/aad2883f41974144adb303ee279587b2, entries=150, sequenceid=353, filesize=12.0 K 2024-12-04T14:59:48,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=87.22 KB/89310 for cf9bed3c8089ea388e8abd4912312aee in 125ms, sequenceid=353, compaction requested=false 2024-12-04T14:59:48,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:48,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=93.93 KB heapSize=265.45 KB 2024-12-04T14:59:48,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:48,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:48,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:48,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:48,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:48,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:48,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/5e7789f308a440d8ab3f6d4495102298 is 50, key is test_row_0/A:col10/1733324388454/Put/seqid=0 2024-12-04T14:59:48,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742112_1288 (size=19621) 2024-12-04T14:59:48,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T14:59:48,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:48,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:48,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:48,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:48,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:48,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324448502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:48,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:48,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:48,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:48,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324448502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:48,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324448611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:48,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324448611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:48,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:48,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:48,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:48,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T14:59:48,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:48,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:48,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:48,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:48,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:48,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T14:59:48,770 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/2fb3843171d744398b3aff4b272a522c as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/2fb3843171d744398b3aff4b272a522c 2024-12-04T14:59:48,774 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/A of cf9bed3c8089ea388e8abd4912312aee into 2fb3843171d744398b3aff4b272a522c(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:48,774 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:48,774 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/A, priority=12, startTime=1733324388324; duration=0sec 2024-12-04T14:59:48,775 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:48,775 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:A 2024-12-04T14:59:48,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T14:59:48,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:48,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:48,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:48,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:48,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:48,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324448820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:48,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324448820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:48,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/5e7789f308a440d8ab3f6d4495102298 2024-12-04T14:59:48,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/e66d14b5f3604e3587978a93fde67755 is 50, key is test_row_0/B:col10/1733324388454/Put/seqid=0 2024-12-04T14:59:48,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742113_1289 (size=12301) 2024-12-04T14:59:48,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T14:59:48,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:48,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:48,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:48,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:48,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T14:59:49,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T14:59:49,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:49,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:49,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:49,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:49,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:49,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:49,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324449130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:49,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:49,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:49,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324449130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:49,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:49,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:49,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:49,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:49,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:49,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T14:59:49,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:49,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:49,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:49,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:49,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:49,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:49,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/e66d14b5f3604e3587978a93fde67755 2024-12-04T14:59:49,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/a12cad326ccf4f66b6811cecbb3a45f0 is 50, key is test_row_0/C:col10/1733324388454/Put/seqid=0 2024-12-04T14:59:49,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742114_1290 (size=12301) 2024-12-04T14:59:49,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T14:59:49,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:49,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:49,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:49,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:49,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:49,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T14:59:49,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T14:59:49,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:49,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:49,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:49,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:49,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:49,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:49,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324449640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:49,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:49,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:49,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324449640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:49,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:49,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:49,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:49,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:49,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:49,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/a12cad326ccf4f66b6811cecbb3a45f0 2024-12-04T14:59:49,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/5e7789f308a440d8ab3f6d4495102298 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/5e7789f308a440d8ab3f6d4495102298 2024-12-04T14:59:49,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/5e7789f308a440d8ab3f6d4495102298, entries=300, sequenceid=386, filesize=19.2 K 2024-12-04T14:59:49,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/e66d14b5f3604e3587978a93fde67755 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/e66d14b5f3604e3587978a93fde67755 2024-12-04T14:59:49,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/e66d14b5f3604e3587978a93fde67755, entries=150, sequenceid=386, filesize=12.0 K 2024-12-04T14:59:49,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/a12cad326ccf4f66b6811cecbb3a45f0 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/a12cad326ccf4f66b6811cecbb3a45f0 2024-12-04T14:59:49,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/a12cad326ccf4f66b6811cecbb3a45f0, entries=150, sequenceid=386, filesize=12.0 K 2024-12-04T14:59:49,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~265.41 KB/271776, currentSize=93.93 KB/96180 for cf9bed3c8089ea388e8abd4912312aee in 1248ms, sequenceid=386, compaction requested=true 2024-12-04T14:59:49,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:49,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:59:49,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:49,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:59:49,703 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:49,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:49,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:59:49,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:49,703 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:49,704 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44837 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:49,704 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:49,704 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/A is initiating minor compaction (all files) 2024-12-04T14:59:49,704 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/B is initiating minor compaction (all files) 2024-12-04T14:59:49,704 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/A in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:49,704 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/B in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:49,704 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/2fb3843171d744398b3aff4b272a522c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e408a276f6054d64bfe113f1e47d514f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/5e7789f308a440d8ab3f6d4495102298] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=43.8 K 2024-12-04T14:59:49,704 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/1d8aac34ea2c4afaaa93f522cb806a4c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/39a1df2348464d34a9cacf3e094d2e72, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/e66d14b5f3604e3587978a93fde67755] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=36.6 K 2024-12-04T14:59:49,705 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fb3843171d744398b3aff4b272a522c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733324387854 2024-12-04T14:59:49,705 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d8aac34ea2c4afaaa93f522cb806a4c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733324387854 2024-12-04T14:59:49,705 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e408a276f6054d64bfe113f1e47d514f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733324388199 2024-12-04T14:59:49,705 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 39a1df2348464d34a9cacf3e094d2e72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733324388199 2024-12-04T14:59:49,705 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e7789f308a440d8ab3f6d4495102298, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733324388422 2024-12-04T14:59:49,705 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting e66d14b5f3604e3587978a93fde67755, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733324388440 2024-12-04T14:59:49,714 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#A#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:49,714 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/0d671c6f092a46129f3842eaab6daa7b is 50, key is test_row_0/A:col10/1733324388454/Put/seqid=0 2024-12-04T14:59:49,714 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#B#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:49,715 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/9b1a25bc4f8e4d03968bd7f442230667 is 50, key is test_row_0/B:col10/1733324388454/Put/seqid=0 2024-12-04T14:59:49,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742115_1291 (size=13017) 2024-12-04T14:59:49,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742116_1292 (size=13017) 2024-12-04T14:59:49,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T14:59:49,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:49,719 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-04T14:59:49,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:49,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:49,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:49,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:49,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:49,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:49,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/13653e1580954a76b94db3d4ab57973c is 50, key is test_row_0/A:col10/1733324388493/Put/seqid=0 2024-12-04T14:59:49,726 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/9b1a25bc4f8e4d03968bd7f442230667 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/9b1a25bc4f8e4d03968bd7f442230667 2024-12-04T14:59:49,730 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/B of cf9bed3c8089ea388e8abd4912312aee into 9b1a25bc4f8e4d03968bd7f442230667(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:49,730 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:49,730 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/B, priority=13, startTime=1733324389703; duration=0sec 2024-12-04T14:59:49,730 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:49,730 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:B 2024-12-04T14:59:49,730 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:49,731 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:49,731 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/C is initiating minor compaction (all files) 2024-12-04T14:59:49,731 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/C in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:49,731 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/b20f8a7e58ff4808a3ba24583d96fdd6, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/aad2883f41974144adb303ee279587b2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/a12cad326ccf4f66b6811cecbb3a45f0] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=36.6 K 2024-12-04T14:59:49,732 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting b20f8a7e58ff4808a3ba24583d96fdd6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733324387854 2024-12-04T14:59:49,732 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting aad2883f41974144adb303ee279587b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733324388199 2024-12-04T14:59:49,732 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting a12cad326ccf4f66b6811cecbb3a45f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733324388440 2024-12-04T14:59:49,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742117_1293 (size=12301) 2024-12-04T14:59:49,739 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#C#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:49,740 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/979913b7729e4f7aa2525293c74eed4b is 50, key is test_row_0/C:col10/1733324388454/Put/seqid=0 2024-12-04T14:59:49,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742118_1294 (size=13017) 2024-12-04T14:59:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:49,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:49,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324449919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:49,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:49,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:49,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:49,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324449921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:49,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:49,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:49,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:49,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324449922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:49,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:49,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:49,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:50,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:50,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324450030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:50,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:50,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324450030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:50,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:50,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:50,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324450030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:50,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:50,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:50,123 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/0d671c6f092a46129f3842eaab6daa7b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/0d671c6f092a46129f3842eaab6daa7b 2024-12-04T14:59:50,128 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/A of cf9bed3c8089ea388e8abd4912312aee into 0d671c6f092a46129f3842eaab6daa7b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:50,128 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:50,128 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/A, priority=13, startTime=1733324389703; duration=0sec 2024-12-04T14:59:50,128 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:50,128 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:A 2024-12-04T14:59:50,133 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/13653e1580954a76b94db3d4ab57973c 2024-12-04T14:59:50,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/93503448702349d4b7412dbfef57815a is 50, key is test_row_0/B:col10/1733324388493/Put/seqid=0 2024-12-04T14:59:50,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742119_1295 (size=12301) 2024-12-04T14:59:50,148 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/979913b7729e4f7aa2525293c74eed4b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/979913b7729e4f7aa2525293c74eed4b 2024-12-04T14:59:50,153 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/C of cf9bed3c8089ea388e8abd4912312aee into 979913b7729e4f7aa2525293c74eed4b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:50,153 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:50,153 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/C, priority=13, startTime=1733324389703; duration=0sec 2024-12-04T14:59:50,153 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:50,153 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:C 2024-12-04T14:59:50,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:50,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324450240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:50,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:50,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:50,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324450240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:50,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324450240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T14:59:50,544 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/93503448702349d4b7412dbfef57815a 2024-12-04T14:59:50,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/bce6de973756473cb9e03e8e3699b5e9 is 50, key is test_row_0/C:col10/1733324388493/Put/seqid=0 2024-12-04T14:59:50,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:50,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324450549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:50,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324450550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:50,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:50,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324450550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:50,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:50,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742120_1296 (size=12301) 2024-12-04T14:59:50,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324450649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:50,652 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_0' in cf9bed3c8089ea388e8abd4912312aee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:50,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324450660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:50,662 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_1' in cf9bed3c8089ea388e8abd4912312aee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:50,954 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/bce6de973756473cb9e03e8e3699b5e9 2024-12-04T14:59:50,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/13653e1580954a76b94db3d4ab57973c as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/13653e1580954a76b94db3d4ab57973c 2024-12-04T14:59:50,961 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/13653e1580954a76b94db3d4ab57973c, entries=150, sequenceid=404, filesize=12.0 K 2024-12-04T14:59:50,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/93503448702349d4b7412dbfef57815a as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/93503448702349d4b7412dbfef57815a 2024-12-04T14:59:50,965 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/93503448702349d4b7412dbfef57815a, entries=150, sequenceid=404, filesize=12.0 K 2024-12-04T14:59:50,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/bce6de973756473cb9e03e8e3699b5e9 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/bce6de973756473cb9e03e8e3699b5e9 2024-12-04T14:59:50,969 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/bce6de973756473cb9e03e8e3699b5e9, entries=150, sequenceid=404, filesize=12.0 K 2024-12-04T14:59:50,970 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for cf9bed3c8089ea388e8abd4912312aee in 1251ms, sequenceid=404, compaction requested=false 2024-12-04T14:59:50,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:50,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:50,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-04T14:59:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-04T14:59:50,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-04T14:59:50,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6350 sec 2024-12-04T14:59:50,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.6430 sec 2024-12-04T14:59:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:51,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-04T14:59:51,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:51,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:51,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:51,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:51,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:51,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:51,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/4fb3119c10354f4191814a54ec870590 is 50, key is test_row_0/A:col10/1733324391060/Put/seqid=0 2024-12-04T14:59:51,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742121_1297 (size=12301) 2024-12-04T14:59:51,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/4fb3119c10354f4191814a54ec870590 2024-12-04T14:59:51,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/40a245aa49af421c8fc83377c8ed7668 is 50, key is test_row_0/B:col10/1733324391060/Put/seqid=0 2024-12-04T14:59:51,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:51,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742122_1298 (size=12301) 2024-12-04T14:59:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324451079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:51,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:51,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/40a245aa49af421c8fc83377c8ed7668 2024-12-04T14:59:51,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324451080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:51,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324451080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:51,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:51,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:51,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/630db658626a4eb680c314ae4622bbf3 is 50, key is test_row_0/C:col10/1733324391060/Put/seqid=0 2024-12-04T14:59:51,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742123_1299 (size=12301) 2024-12-04T14:59:51,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324451190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324451190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324451190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:51,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324451400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:51,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:51,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:51,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324451400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324451400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:51,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:51,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:51,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/630db658626a4eb680c314ae4622bbf3 2024-12-04T14:59:51,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/4fb3119c10354f4191814a54ec870590 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/4fb3119c10354f4191814a54ec870590 2024-12-04T14:59:51,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/4fb3119c10354f4191814a54ec870590, entries=150, sequenceid=425, filesize=12.0 K 2024-12-04T14:59:51,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/40a245aa49af421c8fc83377c8ed7668 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/40a245aa49af421c8fc83377c8ed7668 2024-12-04T14:59:51,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/40a245aa49af421c8fc83377c8ed7668, entries=150, sequenceid=425, filesize=12.0 K 2024-12-04T14:59:51,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/630db658626a4eb680c314ae4622bbf3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/630db658626a4eb680c314ae4622bbf3 2024-12-04T14:59:51,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/630db658626a4eb680c314ae4622bbf3, entries=150, sequenceid=425, filesize=12.0 K 2024-12-04T14:59:51,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for cf9bed3c8089ea388e8abd4912312aee in 452ms, sequenceid=425, compaction requested=true 2024-12-04T14:59:51,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:51,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:59:51,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:51,513 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:51,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:59:51,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:51,513 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:51,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:59:51,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:51,515 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:51,515 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/A is initiating minor compaction (all files) 2024-12-04T14:59:51,515 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:51,515 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/A in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:51,515 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/B is initiating minor compaction (all files) 2024-12-04T14:59:51,515 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/B in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:51,515 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/0d671c6f092a46129f3842eaab6daa7b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/13653e1580954a76b94db3d4ab57973c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/4fb3119c10354f4191814a54ec870590] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=36.7 K 2024-12-04T14:59:51,515 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/9b1a25bc4f8e4d03968bd7f442230667, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/93503448702349d4b7412dbfef57815a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/40a245aa49af421c8fc83377c8ed7668] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=36.7 K 2024-12-04T14:59:51,516 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d671c6f092a46129f3842eaab6daa7b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733324388440 2024-12-04T14:59:51,516 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b1a25bc4f8e4d03968bd7f442230667, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733324388440 2024-12-04T14:59:51,516 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 93503448702349d4b7412dbfef57815a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1733324388493 2024-12-04T14:59:51,516 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13653e1580954a76b94db3d4ab57973c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1733324388493 2024-12-04T14:59:51,516 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 40a245aa49af421c8fc83377c8ed7668, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733324389921 2024-12-04T14:59:51,516 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fb3119c10354f4191814a54ec870590, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733324389921 2024-12-04T14:59:51,523 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#B#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:51,524 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/1ada3fc975d941e395821c2c26cf84be is 50, key is test_row_0/B:col10/1733324391060/Put/seqid=0 2024-12-04T14:59:51,532 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#A#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:51,533 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/e167959e4f084b83bbc9d12a9e124f49 is 50, key is test_row_0/A:col10/1733324391060/Put/seqid=0 2024-12-04T14:59:51,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742124_1300 (size=13119) 2024-12-04T14:59:51,541 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/1ada3fc975d941e395821c2c26cf84be as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/1ada3fc975d941e395821c2c26cf84be 2024-12-04T14:59:51,546 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/B of cf9bed3c8089ea388e8abd4912312aee into 1ada3fc975d941e395821c2c26cf84be(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:51,546 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:51,546 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/B, priority=13, startTime=1733324391513; duration=0sec 2024-12-04T14:59:51,546 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:51,546 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:B 2024-12-04T14:59:51,546 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:51,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742125_1301 (size=13119) 2024-12-04T14:59:51,547 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:51,548 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/C is initiating minor compaction (all files) 2024-12-04T14:59:51,548 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/C in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:51,548 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/979913b7729e4f7aa2525293c74eed4b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/bce6de973756473cb9e03e8e3699b5e9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/630db658626a4eb680c314ae4622bbf3] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=36.7 K 2024-12-04T14:59:51,549 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 979913b7729e4f7aa2525293c74eed4b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733324388440 2024-12-04T14:59:51,549 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting bce6de973756473cb9e03e8e3699b5e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1733324388493 2024-12-04T14:59:51,549 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 630db658626a4eb680c314ae4622bbf3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733324389921 2024-12-04T14:59:51,554 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/e167959e4f084b83bbc9d12a9e124f49 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e167959e4f084b83bbc9d12a9e124f49 2024-12-04T14:59:51,558 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/A of cf9bed3c8089ea388e8abd4912312aee into e167959e4f084b83bbc9d12a9e124f49(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:51,558 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:51,558 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/A, priority=13, startTime=1733324391513; duration=0sec 2024-12-04T14:59:51,559 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:51,559 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:A 2024-12-04T14:59:51,559 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#C#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:51,559 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/54085a1ebe5f48dda7627fd51521d60f is 50, key is test_row_0/C:col10/1733324391060/Put/seqid=0 2024-12-04T14:59:51,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742126_1302 (size=13119) 2024-12-04T14:59:51,581 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/54085a1ebe5f48dda7627fd51521d60f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/54085a1ebe5f48dda7627fd51521d60f 2024-12-04T14:59:51,585 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/C of cf9bed3c8089ea388e8abd4912312aee into 54085a1ebe5f48dda7627fd51521d60f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:51,585 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:51,585 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/C, priority=13, startTime=1733324391513; duration=0sec 2024-12-04T14:59:51,585 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:51,585 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:C 2024-12-04T14:59:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:51,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-04T14:59:51,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:51,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:51,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:51,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:51,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:51,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:51,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/f4d1897b70f644c9965bf380c4e4a5b0 is 50, key is test_row_0/A:col10/1733324391712/Put/seqid=0 2024-12-04T14:59:51,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742127_1303 (size=14741) 2024-12-04T14:59:51,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324451728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:51,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324451730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:51,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324451731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:51,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:51,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324451840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:51,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:51,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324451841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:51,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:51,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:51,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324451841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:51,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:51,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:52,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:52,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324452050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:52,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324452050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:52,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324452050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:52,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/f4d1897b70f644c9965bf380c4e4a5b0 2024-12-04T14:59:52,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/b6a0c6715f9b4322b5886da634f7b17e is 50, key is test_row_0/B:col10/1733324391712/Put/seqid=0 2024-12-04T14:59:52,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742128_1304 (size=12301) 2024-12-04T14:59:52,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324452360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:52,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324452361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324452361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T14:59:52,500 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:52,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:59:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-04T14:59:52,502 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:59:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T14:59:52,503 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:59:52,503 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:59:52,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/b6a0c6715f9b4322b5886da634f7b17e 2024-12-04T14:59:52,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/7b2a0607f85a4823a6610884466173b5 is 50, key is test_row_0/C:col10/1733324391712/Put/seqid=0 2024-12-04T14:59:52,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742129_1305 (size=12301) 2024-12-04T14:59:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T14:59:52,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T14:59:52,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:52,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:52,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:52,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:52,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:52,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:52,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1733324452670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1733324452670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:52,673 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_0' in cf9bed3c8089ea388e8abd4912312aee of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:52,673 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_1' in cf9bed3c8089ea388e8abd4912312aee of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:52,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T14:59:52,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:52,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:52,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:52,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:52,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:59:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T14:59:52,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324452870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:52,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324452870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:52,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:52,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324452871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:52,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:52,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:52,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:52,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/7b2a0607f85a4823a6610884466173b5 2024-12-04T14:59:52,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/f4d1897b70f644c9965bf380c4e4a5b0 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/f4d1897b70f644c9965bf380c4e4a5b0 2024-12-04T14:59:52,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/f4d1897b70f644c9965bf380c4e4a5b0, entries=200, sequenceid=446, filesize=14.4 K 2024-12-04T14:59:52,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/b6a0c6715f9b4322b5886da634f7b17e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/b6a0c6715f9b4322b5886da634f7b17e 2024-12-04T14:59:52,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/b6a0c6715f9b4322b5886da634f7b17e, entries=150, sequenceid=446, filesize=12.0 K 2024-12-04T14:59:52,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/7b2a0607f85a4823a6610884466173b5 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/7b2a0607f85a4823a6610884466173b5 2024-12-04T14:59:52,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/7b2a0607f85a4823a6610884466173b5, entries=150, sequenceid=446, filesize=12.0 K 2024-12-04T14:59:52,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for cf9bed3c8089ea388e8abd4912312aee in 1244ms, sequenceid=446, compaction requested=false 2024-12-04T14:59:52,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:52,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T14:59:52,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:52,962 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-04T14:59:52,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:52,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:52,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:52,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:52,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:52,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:52,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/5ec5294f50f840adac25ced84f33894a is 50, key is test_row_0/A:col10/1733324391726/Put/seqid=0 2024-12-04T14:59:52,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742130_1306 (size=12301) 2024-12-04T14:59:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T14:59:53,226 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T14:59:53,379 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/5ec5294f50f840adac25ced84f33894a 2024-12-04T14:59:53,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/820b85e367354480acf5e67e31335870 is 50, key is test_row_0/B:col10/1733324391726/Put/seqid=0 2024-12-04T14:59:53,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742131_1307 (size=12301) 2024-12-04T14:59:53,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T14:59:53,804 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/820b85e367354480acf5e67e31335870 2024-12-04T14:59:53,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/9de0771ab6444c77bfc6b95bbe246b2b is 50, key is test_row_0/C:col10/1733324391726/Put/seqid=0 2024-12-04T14:59:53,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742132_1308 (size=12301) 2024-12-04T14:59:53,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:53,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324453904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:53,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324453908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:53,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324453908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:54,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324454010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:54,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324454020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:54,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:54,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324454020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:54,213 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/9de0771ab6444c77bfc6b95bbe246b2b 2024-12-04T14:59:54,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/5ec5294f50f840adac25ced84f33894a as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/5ec5294f50f840adac25ced84f33894a 2024-12-04T14:59:54,220 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/5ec5294f50f840adac25ced84f33894a, entries=150, sequenceid=460, filesize=12.0 K 2024-12-04T14:59:54,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/820b85e367354480acf5e67e31335870 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/820b85e367354480acf5e67e31335870 2024-12-04T14:59:54,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324454221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:54,225 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/820b85e367354480acf5e67e31335870, entries=150, sequenceid=460, filesize=12.0 K 2024-12-04T14:59:54,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/9de0771ab6444c77bfc6b95bbe246b2b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/9de0771ab6444c77bfc6b95bbe246b2b 2024-12-04T14:59:54,229 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/9de0771ab6444c77bfc6b95bbe246b2b, entries=150, sequenceid=460, filesize=12.0 K 2024-12-04T14:59:54,230 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for cf9bed3c8089ea388e8abd4912312aee in 1269ms, sequenceid=460, compaction requested=true 2024-12-04T14:59:54,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:54,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:54,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-04T14:59:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-04T14:59:54,233 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-04T14:59:54,233 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7290 sec 2024-12-04T14:59:54,235 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.7330 sec 2024-12-04T14:59:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:54,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-12-04T14:59:54,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:54,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:54,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:54,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:54,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:54,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:54,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/3a1be435554a41c7ba34d5f2d14b8b88 is 50, key is test_row_0/A:col10/1733324394236/Put/seqid=0 2024-12-04T14:59:54,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742133_1309 (size=14741) 2024-12-04T14:59:54,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/3a1be435554a41c7ba34d5f2d14b8b88 2024-12-04T14:59:54,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/99d2f911cb194f24b36c1e953c981410 is 50, key is test_row_0/B:col10/1733324394236/Put/seqid=0 2024-12-04T14:59:54,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324454261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:54,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324454261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:54,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:54,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:54,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742134_1310 (size=12301) 2024-12-04T14:59:54,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/99d2f911cb194f24b36c1e953c981410 2024-12-04T14:59:54,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/0eaef9e75e8d484b916f534e64a99993 is 50, key is test_row_0/C:col10/1733324394236/Put/seqid=0 2024-12-04T14:59:54,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742135_1311 (size=12301) 2024-12-04T14:59:54,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324454370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324454370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:54,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324454530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:54,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:54,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324454580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:54,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324454580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T14:59:54,651 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:54,651 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:59:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-04T14:59:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T14:59:54,654 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:59:54,655 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:59:54,655 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:59:54,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/0eaef9e75e8d484b916f534e64a99993 2024-12-04T14:59:54,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/3a1be435554a41c7ba34d5f2d14b8b88 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/3a1be435554a41c7ba34d5f2d14b8b88 2024-12-04T14:59:54,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/3a1be435554a41c7ba34d5f2d14b8b88, entries=200, sequenceid=481, filesize=14.4 K 2024-12-04T14:59:54,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/99d2f911cb194f24b36c1e953c981410 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/99d2f911cb194f24b36c1e953c981410 2024-12-04T14:59:54,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/99d2f911cb194f24b36c1e953c981410, entries=150, sequenceid=481, filesize=12.0 K 2024-12-04T14:59:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/0eaef9e75e8d484b916f534e64a99993 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/0eaef9e75e8d484b916f534e64a99993 2024-12-04T14:59:54,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/0eaef9e75e8d484b916f534e64a99993, entries=150, sequenceid=481, filesize=12.0 K 2024-12-04T14:59:54,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for cf9bed3c8089ea388e8abd4912312aee in 460ms, sequenceid=481, compaction requested=true 2024-12-04T14:59:54,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:54,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:59:54,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:54,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:59:54,696 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:59:54,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:54,696 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:59:54,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:59:54,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:54,697 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54902 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:59:54,697 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/A is initiating minor compaction (all files) 2024-12-04T14:59:54,697 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/A in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:54,697 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e167959e4f084b83bbc9d12a9e124f49, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/f4d1897b70f644c9965bf380c4e4a5b0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/5ec5294f50f840adac25ced84f33894a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/3a1be435554a41c7ba34d5f2d14b8b88] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=53.6 K 2024-12-04T14:59:54,698 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e167959e4f084b83bbc9d12a9e124f49, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733324389921 2024-12-04T14:59:54,698 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:59:54,698 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/B is initiating minor compaction (all files) 2024-12-04T14:59:54,698 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/B in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:54,698 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4d1897b70f644c9965bf380c4e4a5b0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733324391077 2024-12-04T14:59:54,698 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/1ada3fc975d941e395821c2c26cf84be, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/b6a0c6715f9b4322b5886da634f7b17e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/820b85e367354480acf5e67e31335870, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/99d2f911cb194f24b36c1e953c981410] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=48.8 K 2024-12-04T14:59:54,698 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ec5294f50f840adac25ced84f33894a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1733324391726 2024-12-04T14:59:54,698 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ada3fc975d941e395821c2c26cf84be, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733324389921 2024-12-04T14:59:54,699 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a1be435554a41c7ba34d5f2d14b8b88, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733324393901 2024-12-04T14:59:54,699 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting b6a0c6715f9b4322b5886da634f7b17e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733324391077 2024-12-04T14:59:54,699 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 820b85e367354480acf5e67e31335870, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1733324391726 2024-12-04T14:59:54,699 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 99d2f911cb194f24b36c1e953c981410, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733324393906 2024-12-04T14:59:54,708 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#A#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:54,708 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#B#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:54,708 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/92a2961f320f458f89c1d6dd3f120fb2 is 50, key is test_row_0/B:col10/1733324394236/Put/seqid=0 2024-12-04T14:59:54,708 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/e6426b95566c468d974744d5e45fea1e is 50, key is test_row_0/A:col10/1733324394236/Put/seqid=0 2024-12-04T14:59:54,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742137_1313 (size=13255) 2024-12-04T14:59:54,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742136_1312 (size=13255) 2024-12-04T14:59:54,726 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/92a2961f320f458f89c1d6dd3f120fb2 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/92a2961f320f458f89c1d6dd3f120fb2 2024-12-04T14:59:54,730 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/B of cf9bed3c8089ea388e8abd4912312aee into 92a2961f320f458f89c1d6dd3f120fb2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:54,731 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:54,731 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/B, priority=12, startTime=1733324394696; duration=0sec 2024-12-04T14:59:54,731 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:54,731 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:B 2024-12-04T14:59:54,731 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:59:54,732 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:59:54,732 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/C is initiating minor compaction (all files) 2024-12-04T14:59:54,732 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/C in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:54,732 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/54085a1ebe5f48dda7627fd51521d60f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/7b2a0607f85a4823a6610884466173b5, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/9de0771ab6444c77bfc6b95bbe246b2b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/0eaef9e75e8d484b916f534e64a99993] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=48.8 K 2024-12-04T14:59:54,732 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 54085a1ebe5f48dda7627fd51521d60f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733324389921 2024-12-04T14:59:54,733 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b2a0607f85a4823a6610884466173b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733324391077 2024-12-04T14:59:54,733 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 9de0771ab6444c77bfc6b95bbe246b2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1733324391726 2024-12-04T14:59:54,733 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eaef9e75e8d484b916f534e64a99993, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733324393906 2024-12-04T14:59:54,741 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#C#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:54,742 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/f5f021830b484a278655b5e2554d3168 is 50, key is test_row_0/C:col10/1733324394236/Put/seqid=0 2024-12-04T14:59:54,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742138_1314 (size=13255) 2024-12-04T14:59:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T14:59:54,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-04T14:59:54,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:54,808 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T14:59:54,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:54,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:54,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:54,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:54,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:54,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:54,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/78b184f7255a46bd965111515dffd849 is 50, key is test_row_0/A:col10/1733324394249/Put/seqid=0 2024-12-04T14:59:54,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742139_1315 (size=12301) 2024-12-04T14:59:54,822 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/78b184f7255a46bd965111515dffd849 2024-12-04T14:59:54,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/4e470fe51a934bcbb5d91173e81c7771 is 50, key is test_row_0/B:col10/1733324394249/Put/seqid=0 2024-12-04T14:59:54,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742140_1316 (size=12301) 2024-12-04T14:59:54,839 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/4e470fe51a934bcbb5d91173e81c7771 2024-12-04T14:59:54,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/d9d883b7645046b2827b03c8b5739ade is 50, key is test_row_0/C:col10/1733324394249/Put/seqid=0 2024-12-04T14:59:54,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742141_1317 (size=12301) 2024-12-04T14:59:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:54,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. as already flushing 2024-12-04T14:59:54,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324394913, max timestamp=1733324394927Num uniques 859; 2024-12-04T14:59:54,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:59:54,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324394913, max timestamp=1733324394927Num uniques 859; 2024-12-04T14:59:54,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:59:54,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324394913, max timestamp=1733324394927Num uniques 859; 2024-12-04T14:59:54,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T14:59:54,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324454964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T14:59:54,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324454965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:55,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324455040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:55,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324455081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:55,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:55,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324455081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:55,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:55,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:55,126 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/e6426b95566c468d974744d5e45fea1e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e6426b95566c468d974744d5e45fea1e 2024-12-04T14:59:55,131 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/A of cf9bed3c8089ea388e8abd4912312aee into e6426b95566c468d974744d5e45fea1e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:55,131 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:55,131 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/A, priority=12, startTime=1733324394696; duration=0sec 2024-12-04T14:59:55,131 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:55,131 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:A 2024-12-04T14:59:55,151 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/f5f021830b484a278655b5e2554d3168 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/f5f021830b484a278655b5e2554d3168 2024-12-04T14:59:55,156 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/C of cf9bed3c8089ea388e8abd4912312aee into f5f021830b484a278655b5e2554d3168(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:55,156 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:55,156 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/C, priority=12, startTime=1733324394696; duration=0sec 2024-12-04T14:59:55,156 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:55,156 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:C 2024-12-04T14:59:55,256 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/d9d883b7645046b2827b03c8b5739ade 2024-12-04T14:59:55,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/78b184f7255a46bd965111515dffd849 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/78b184f7255a46bd965111515dffd849 2024-12-04T14:59:55,273 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/78b184f7255a46bd965111515dffd849, entries=150, sequenceid=493, filesize=12.0 K 2024-12-04T14:59:55,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/4e470fe51a934bcbb5d91173e81c7771 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/4e470fe51a934bcbb5d91173e81c7771 2024-12-04T14:59:55,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T14:59:55,282 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/4e470fe51a934bcbb5d91173e81c7771, entries=150, sequenceid=493, filesize=12.0 K 2024-12-04T14:59:55,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/d9d883b7645046b2827b03c8b5739ade as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/d9d883b7645046b2827b03c8b5739ade 2024-12-04T14:59:55,287 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/d9d883b7645046b2827b03c8b5739ade, entries=150, sequenceid=493, filesize=12.0 K 2024-12-04T14:59:55,288 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cf9bed3c8089ea388e8abd4912312aee in 480ms, sequenceid=493, compaction requested=false 2024-12-04T14:59:55,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:55,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:55,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-04T14:59:55,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-04T14:59:55,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-04T14:59:55,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 634 msec 2024-12-04T14:59:55,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 640 msec 2024-12-04T14:59:55,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:55,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-04T14:59:55,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:55,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:55,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:55,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:55,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:55,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T14:59:55,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/e57f485022f54d99a2e82c75c1565ace is 50, key is test_row_0/A:col10/1733324394963/Put/seqid=0 2024-12-04T14:59:55,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742142_1318 (size=14741) 2024-12-04T14:59:55,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:55,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324455308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:55,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:55,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324455309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:55,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324455420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:55,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:55,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324455421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:55,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:55,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:55,426 INFO [Thread-1080 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1080. 2024-12-04T14:59:55,426 DEBUG [Thread-1080 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:55,426 DEBUG [Thread-1080 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:55,426 DEBUG [Thread-1080 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:55,426 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:55,428 INFO [Thread-1086 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1086. 2024-12-04T14:59:55,428 DEBUG [Thread-1086 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:55,428 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:55,428 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:55,429 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:55,429 INFO [Thread-1084 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1084. 2024-12-04T14:59:55,429 DEBUG [Thread-1084 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:55,429 DEBUG [Thread-1084 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:55,429 DEBUG [Thread-1084 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:55,429 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:55,430 INFO [Thread-1088 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1088. 2024-12-04T14:59:55,430 DEBUG [Thread-1088 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:55,430 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:55,430 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:55,430 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:55,430 INFO [Thread-1082 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1082. 2024-12-04T14:59:55,430 DEBUG [Thread-1082 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:55,430 DEBUG [Thread-1082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:55,431 DEBUG [Thread-1082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:55,431 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:55,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:55,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324455631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324455631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:55,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:55,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:55,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/e57f485022f54d99a2e82c75c1565ace 2024-12-04T14:59:55,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/00eb91da927944b38542c9d7852abfbe is 50, key is test_row_0/B:col10/1733324394963/Put/seqid=0 2024-12-04T14:59:55,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742143_1319 (size=12301) 2024-12-04T14:59:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T14:59:55,790 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:55,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:55,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324455940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324455940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:55,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:55,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:55,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:56,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1733324456050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:56,052 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in cf9bed3c8089ea388e8abd4912312aee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2147 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:59:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:56,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/00eb91da927944b38542c9d7852abfbe 2024-12-04T14:59:56,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/486ba5ea42f644b89dc0d8ce768c6c18 is 50, key is test_row_0/C:col10/1733324394963/Put/seqid=0 2024-12-04T14:59:56,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742144_1320 (size=12301) 2024-12-04T14:59:56,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:56,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:59:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57358 deadline: 1733324456450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1733324456451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:56,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:56,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:56,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:56,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:59:56,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:56,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:59:56,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/486ba5ea42f644b89dc0d8ce768c6c18 2024-12-04T14:59:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/e57f485022f54d99a2e82c75c1565ace as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e57f485022f54d99a2e82c75c1565ace 2024-12-04T14:59:56,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e57f485022f54d99a2e82c75c1565ace, entries=200, sequenceid=532, filesize=14.4 K 2024-12-04T14:59:56,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/00eb91da927944b38542c9d7852abfbe as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/00eb91da927944b38542c9d7852abfbe 2024-12-04T14:59:56,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/00eb91da927944b38542c9d7852abfbe, entries=150, sequenceid=532, filesize=12.0 K 2024-12-04T14:59:56,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/486ba5ea42f644b89dc0d8ce768c6c18 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/486ba5ea42f644b89dc0d8ce768c6c18 2024-12-04T14:59:56,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/486ba5ea42f644b89dc0d8ce768c6c18, entries=150, sequenceid=532, filesize=12.0 K 2024-12-04T14:59:56,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for cf9bed3c8089ea388e8abd4912312aee in 1255ms, sequenceid=532, compaction requested=true 2024-12-04T14:59:56,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:56,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:59:56,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:56,549 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:56,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:59:56,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:56,549 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:56,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf9bed3c8089ea388e8abd4912312aee:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:59:56,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:56,549 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:56,549 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:56,549 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/B is initiating minor compaction (all files) 2024-12-04T14:59:56,549 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/A is initiating minor compaction (all files) 2024-12-04T14:59:56,549 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/A in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:56,549 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/B in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:56,549 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e6426b95566c468d974744d5e45fea1e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/78b184f7255a46bd965111515dffd849, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e57f485022f54d99a2e82c75c1565ace] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=39.4 K 2024-12-04T14:59:56,549 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/92a2961f320f458f89c1d6dd3f120fb2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/4e470fe51a934bcbb5d91173e81c7771, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/00eb91da927944b38542c9d7852abfbe] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=37.0 K 2024-12-04T14:59:56,550 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6426b95566c468d974744d5e45fea1e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733324393906 2024-12-04T14:59:56,550 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 92a2961f320f458f89c1d6dd3f120fb2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733324393906 2024-12-04T14:59:56,550 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78b184f7255a46bd965111515dffd849, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1733324394249 2024-12-04T14:59:56,550 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e470fe51a934bcbb5d91173e81c7771, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1733324394249 2024-12-04T14:59:56,550 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e57f485022f54d99a2e82c75c1565ace, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1733324394957 2024-12-04T14:59:56,550 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 00eb91da927944b38542c9d7852abfbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1733324394957 2024-12-04T14:59:56,556 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#B#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:56,556 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/e93f64eea4ef4ff3b6daf14d35a2ac57 is 50, key is test_row_0/B:col10/1733324394963/Put/seqid=0 2024-12-04T14:59:56,557 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#A#compaction#277 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:56,557 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/fc9752e581e845498d510397fcf5ff26 is 50, key is test_row_0/A:col10/1733324394963/Put/seqid=0 2024-12-04T14:59:56,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742145_1321 (size=13357) 2024-12-04T14:59:56,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742146_1322 (size=13357) 2024-12-04T14:59:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:56,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T14:59:56,682 INFO [Thread-1073 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1073. 2024-12-04T14:59:56,682 DEBUG [Thread-1073 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:56,682 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:56,682 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:56,682 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:56,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:56,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:56,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:56,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:56,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:56,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:56,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/0510541118df4d22b3e127418e55f749 is 50, key is test_row_0/A:col10/1733324395304/Put/seqid=0 2024-12-04T14:59:56,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742147_1323 (size=12301) 2024-12-04T14:59:56,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=543 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/0510541118df4d22b3e127418e55f749 2024-12-04T14:59:56,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/1c08b65c68bc491682a3a61d8a1790fa is 50, key is test_row_0/B:col10/1733324395304/Put/seqid=0 2024-12-04T14:59:56,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742148_1324 (size=12301) 2024-12-04T14:59:56,712 INFO [Thread-1071 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1071. 2024-12-04T14:59:56,712 DEBUG [Thread-1071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:56,712 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:56,712 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:56,712 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:56,970 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/e93f64eea4ef4ff3b6daf14d35a2ac57 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/e93f64eea4ef4ff3b6daf14d35a2ac57 2024-12-04T14:59:56,970 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/fc9752e581e845498d510397fcf5ff26 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/fc9752e581e845498d510397fcf5ff26 2024-12-04T14:59:56,976 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/A of cf9bed3c8089ea388e8abd4912312aee into fc9752e581e845498d510397fcf5ff26(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:56,976 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:56,976 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/B of cf9bed3c8089ea388e8abd4912312aee into e93f64eea4ef4ff3b6daf14d35a2ac57(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:56,976 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/A, priority=13, startTime=1733324396548; duration=0sec 2024-12-04T14:59:56,976 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:56,976 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/B, priority=13, startTime=1733324396549; duration=0sec 2024-12-04T14:59:56,976 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:59:56,976 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:A 2024-12-04T14:59:56,976 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:59:56,976 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:56,976 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:B 2024-12-04T14:59:56,977 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:59:56,977 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): cf9bed3c8089ea388e8abd4912312aee/C is initiating minor compaction (all files) 2024-12-04T14:59:56,977 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf9bed3c8089ea388e8abd4912312aee/C in TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:56,977 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/f5f021830b484a278655b5e2554d3168, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/d9d883b7645046b2827b03c8b5739ade, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/486ba5ea42f644b89dc0d8ce768c6c18] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp, totalSize=37.0 K 2024-12-04T14:59:56,978 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5f021830b484a278655b5e2554d3168, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733324393906 2024-12-04T14:59:56,978 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9d883b7645046b2827b03c8b5739ade, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1733324394249 2024-12-04T14:59:56,978 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 486ba5ea42f644b89dc0d8ce768c6c18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1733324394957 2024-12-04T14:59:56,987 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf9bed3c8089ea388e8abd4912312aee#C#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:59:56,987 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/c848679f232149b09859625c09cbe329 is 50, key is test_row_0/C:col10/1733324394963/Put/seqid=0 2024-12-04T14:59:56,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742149_1325 (size=13357) 2024-12-04T14:59:57,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=543 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/1c08b65c68bc491682a3a61d8a1790fa 2024-12-04T14:59:57,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/7b8d76087b4c4e5da0d3a3f8d8955cfd is 50, key is test_row_0/C:col10/1733324395304/Put/seqid=0 2024-12-04T14:59:57,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742150_1326 (size=12301) 2024-12-04T14:59:57,396 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/c848679f232149b09859625c09cbe329 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/c848679f232149b09859625c09cbe329 2024-12-04T14:59:57,403 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf9bed3c8089ea388e8abd4912312aee/C of cf9bed3c8089ea388e8abd4912312aee into c848679f232149b09859625c09cbe329(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:59:57,403 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:57,403 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., storeName=cf9bed3c8089ea388e8abd4912312aee/C, priority=13, startTime=1733324396549; duration=0sec 2024-12-04T14:59:57,403 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:59:57,403 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf9bed3c8089ea388e8abd4912312aee:C 2024-12-04T14:59:57,461 INFO [Thread-1075 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1075. 2024-12-04T14:59:57,461 DEBUG [Thread-1075 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:57,461 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:57,461 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:57,461 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:57,471 INFO [Thread-1069 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1069. 2024-12-04T14:59:57,471 DEBUG [Thread-1069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:57,471 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:57,472 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:57,472 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:57,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=543 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/7b8d76087b4c4e5da0d3a3f8d8955cfd 2024-12-04T14:59:57,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/0510541118df4d22b3e127418e55f749 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/0510541118df4d22b3e127418e55f749 2024-12-04T14:59:57,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/0510541118df4d22b3e127418e55f749, entries=150, sequenceid=543, filesize=12.0 K 2024-12-04T14:59:57,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/1c08b65c68bc491682a3a61d8a1790fa as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/1c08b65c68bc491682a3a61d8a1790fa 2024-12-04T14:59:57,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/1c08b65c68bc491682a3a61d8a1790fa, entries=150, sequenceid=543, filesize=12.0 K 2024-12-04T14:59:57,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/7b8d76087b4c4e5da0d3a3f8d8955cfd as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/7b8d76087b4c4e5da0d3a3f8d8955cfd 2024-12-04T14:59:57,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/7b8d76087b4c4e5da0d3a3f8d8955cfd, entries=150, sequenceid=543, filesize=12.0 K 2024-12-04T14:59:57,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=20.13 KB/20610 for cf9bed3c8089ea388e8abd4912312aee in 856ms, sequenceid=543, compaction requested=false 2024-12-04T14:59:57,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf9bed3c8089ea388e8abd4912312aee: 2024-12-04T14:59:58,072 INFO [Thread-1077 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1077. 2024-12-04T14:59:58,072 DEBUG [Thread-1077 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:58,072 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:58,072 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:58,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:58,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T14:59:58,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-12-04T14:59:58,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-12-04T14:59:58,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-12-04T14:59:58,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-12-04T14:59:58,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-12-04T14:59:58,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T14:59:58,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4017 2024-12-04T14:59:58,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3792 2024-12-04T14:59:58,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3836 2024-12-04T14:59:58,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4007 2024-12-04T14:59:58,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3800 2024-12-04T14:59:58,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T14:59:58,073 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T14:59:58,073 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T14:59:58,073 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:58,073 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:58,073 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:58,074 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T14:59:58,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T14:59:58,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T14:59:58,077 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324398077"}]},"ts":"1733324398077"} 2024-12-04T14:59:58,079 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T14:59:58,079 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T14:59:58,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T14:59:58,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf9bed3c8089ea388e8abd4912312aee, UNASSIGN}] 2024-12-04T14:59:58,082 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf9bed3c8089ea388e8abd4912312aee, UNASSIGN 2024-12-04T14:59:58,083 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=cf9bed3c8089ea388e8abd4912312aee, regionState=CLOSING, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T14:59:58,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf9bed3c8089ea388e8abd4912312aee, UNASSIGN because future has completed 2024-12-04T14:59:58,085 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T14:59:58,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801}] 2024-12-04T14:59:58,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T14:59:58,238 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(122): Close cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:58,238 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T14:59:58,238 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1722): Closing cf9bed3c8089ea388e8abd4912312aee, disabling compactions & flushes 2024-12-04T14:59:58,238 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:58,238 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:58,238 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. after waiting 0 ms 2024-12-04T14:59:58,238 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:58,239 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(2902): Flushing cf9bed3c8089ea388e8abd4912312aee 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-04T14:59:58,239 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=A 2024-12-04T14:59:58,239 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:58,239 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=B 2024-12-04T14:59:58,239 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:58,239 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf9bed3c8089ea388e8abd4912312aee, store=C 2024-12-04T14:59:58,239 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T14:59:58,243 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/5e3fc5f26650434c823fa5af312738e3 is 50, key is test_row_0/A:col10/1733324397470/Put/seqid=0 2024-12-04T14:59:58,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742151_1327 (size=12301) 2024-12-04T14:59:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T14:59:58,647 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=553 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/5e3fc5f26650434c823fa5af312738e3 2024-12-04T14:59:58,655 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/2d27f68618154a68a5a890f3cfecae56 is 50, key is test_row_0/B:col10/1733324397470/Put/seqid=0 2024-12-04T14:59:58,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742152_1328 (size=12301) 2024-12-04T14:59:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T14:59:59,060 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=553 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/2d27f68618154a68a5a890f3cfecae56 2024-12-04T14:59:59,068 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/36f8af5c462c48f786f44e09a8fa2aa0 is 50, key is test_row_0/C:col10/1733324397470/Put/seqid=0 2024-12-04T14:59:59,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742153_1329 (size=12301) 2024-12-04T14:59:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T14:59:59,473 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=553 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/36f8af5c462c48f786f44e09a8fa2aa0 2024-12-04T14:59:59,477 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/A/5e3fc5f26650434c823fa5af312738e3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/5e3fc5f26650434c823fa5af312738e3 2024-12-04T14:59:59,480 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/5e3fc5f26650434c823fa5af312738e3, entries=150, sequenceid=553, filesize=12.0 K 2024-12-04T14:59:59,481 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/B/2d27f68618154a68a5a890f3cfecae56 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/2d27f68618154a68a5a890f3cfecae56 2024-12-04T14:59:59,485 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/2d27f68618154a68a5a890f3cfecae56, entries=150, sequenceid=553, filesize=12.0 K 2024-12-04T14:59:59,486 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/.tmp/C/36f8af5c462c48f786f44e09a8fa2aa0 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/36f8af5c462c48f786f44e09a8fa2aa0 2024-12-04T14:59:59,490 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/36f8af5c462c48f786f44e09a8fa2aa0, entries=150, sequenceid=553, filesize=12.0 K 2024-12-04T14:59:59,491 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for cf9bed3c8089ea388e8abd4912312aee in 1253ms, sequenceid=553, compaction requested=true 2024-12-04T14:59:59,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/2e36d62ec39e45cabaeadd8efa4c1fe4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/b98c0fb81b0a4ae28d3fa5d4a401a6db, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e8c3247d690c4c2891fec7db940fb2eb, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/b270e22452294d169e2d70ffcce444f0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/885f406099d141c6b22274f98d5351eb, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/32c7180e1f8e4e77b27b5fee54b2bb39, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/f661fc9c9c724851a2060722cba61394, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/237166f05886495b9ad0467b397db2ec, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/25a9edbc18b142f5b8b8c4a4740e6cbb, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/595b6be5603f4c118af529631d74a897, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/3398e1f6267f46b5a6103bc41bdbb816, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/ca4148a1672f453ebb9ae0630eb40226, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/12c5df6a6aaf42deb5755fa1ed42ac52, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/b8c52dfff60e47a3affd5b48933b512d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/baf4660076e2411a8376234d408feb33, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/28807ab9f48a4ab8a5ffcad216523483, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/8f4d64dc856a44a18d6c5af6845db1d4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/2fb3843171d744398b3aff4b272a522c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/d183fdf678254d73a1b0151114b33af3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e408a276f6054d64bfe113f1e47d514f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/5e7789f308a440d8ab3f6d4495102298, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/0d671c6f092a46129f3842eaab6daa7b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/13653e1580954a76b94db3d4ab57973c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e167959e4f084b83bbc9d12a9e124f49, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/4fb3119c10354f4191814a54ec870590, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/f4d1897b70f644c9965bf380c4e4a5b0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/5ec5294f50f840adac25ced84f33894a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/3a1be435554a41c7ba34d5f2d14b8b88, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e6426b95566c468d974744d5e45fea1e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/78b184f7255a46bd965111515dffd849, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e57f485022f54d99a2e82c75c1565ace] to archive 2024-12-04T14:59:59,492 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:59:59,493 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/2e36d62ec39e45cabaeadd8efa4c1fe4 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/2e36d62ec39e45cabaeadd8efa4c1fe4 2024-12-04T14:59:59,494 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/b98c0fb81b0a4ae28d3fa5d4a401a6db to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/b98c0fb81b0a4ae28d3fa5d4a401a6db 2024-12-04T14:59:59,496 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e8c3247d690c4c2891fec7db940fb2eb to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e8c3247d690c4c2891fec7db940fb2eb 2024-12-04T14:59:59,497 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/b270e22452294d169e2d70ffcce444f0 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/b270e22452294d169e2d70ffcce444f0 2024-12-04T14:59:59,497 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/885f406099d141c6b22274f98d5351eb to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/885f406099d141c6b22274f98d5351eb 2024-12-04T14:59:59,498 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/32c7180e1f8e4e77b27b5fee54b2bb39 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/32c7180e1f8e4e77b27b5fee54b2bb39 2024-12-04T14:59:59,499 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/f661fc9c9c724851a2060722cba61394 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/f661fc9c9c724851a2060722cba61394 2024-12-04T14:59:59,500 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/237166f05886495b9ad0467b397db2ec to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/237166f05886495b9ad0467b397db2ec 2024-12-04T14:59:59,501 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/25a9edbc18b142f5b8b8c4a4740e6cbb to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/25a9edbc18b142f5b8b8c4a4740e6cbb 2024-12-04T14:59:59,502 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/595b6be5603f4c118af529631d74a897 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/595b6be5603f4c118af529631d74a897 2024-12-04T14:59:59,503 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/3398e1f6267f46b5a6103bc41bdbb816 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/3398e1f6267f46b5a6103bc41bdbb816 2024-12-04T14:59:59,504 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/ca4148a1672f453ebb9ae0630eb40226 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/ca4148a1672f453ebb9ae0630eb40226 2024-12-04T14:59:59,505 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/12c5df6a6aaf42deb5755fa1ed42ac52 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/12c5df6a6aaf42deb5755fa1ed42ac52 2024-12-04T14:59:59,506 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/b8c52dfff60e47a3affd5b48933b512d to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/b8c52dfff60e47a3affd5b48933b512d 2024-12-04T14:59:59,506 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/baf4660076e2411a8376234d408feb33 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/baf4660076e2411a8376234d408feb33 2024-12-04T14:59:59,507 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/28807ab9f48a4ab8a5ffcad216523483 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/28807ab9f48a4ab8a5ffcad216523483 2024-12-04T14:59:59,508 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/8f4d64dc856a44a18d6c5af6845db1d4 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/8f4d64dc856a44a18d6c5af6845db1d4 2024-12-04T14:59:59,509 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/2fb3843171d744398b3aff4b272a522c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/2fb3843171d744398b3aff4b272a522c 2024-12-04T14:59:59,510 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/d183fdf678254d73a1b0151114b33af3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/d183fdf678254d73a1b0151114b33af3 2024-12-04T14:59:59,511 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e408a276f6054d64bfe113f1e47d514f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e408a276f6054d64bfe113f1e47d514f 2024-12-04T14:59:59,512 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/5e7789f308a440d8ab3f6d4495102298 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/5e7789f308a440d8ab3f6d4495102298 2024-12-04T14:59:59,513 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/0d671c6f092a46129f3842eaab6daa7b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/0d671c6f092a46129f3842eaab6daa7b 2024-12-04T14:59:59,513 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/13653e1580954a76b94db3d4ab57973c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/13653e1580954a76b94db3d4ab57973c 2024-12-04T14:59:59,514 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e167959e4f084b83bbc9d12a9e124f49 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e167959e4f084b83bbc9d12a9e124f49 2024-12-04T14:59:59,515 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/4fb3119c10354f4191814a54ec870590 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/4fb3119c10354f4191814a54ec870590 2024-12-04T14:59:59,516 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/f4d1897b70f644c9965bf380c4e4a5b0 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/f4d1897b70f644c9965bf380c4e4a5b0 2024-12-04T14:59:59,516 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/5ec5294f50f840adac25ced84f33894a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/5ec5294f50f840adac25ced84f33894a 2024-12-04T14:59:59,517 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/3a1be435554a41c7ba34d5f2d14b8b88 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/3a1be435554a41c7ba34d5f2d14b8b88 2024-12-04T14:59:59,518 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e6426b95566c468d974744d5e45fea1e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e6426b95566c468d974744d5e45fea1e 2024-12-04T14:59:59,519 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/78b184f7255a46bd965111515dffd849 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/78b184f7255a46bd965111515dffd849 2024-12-04T14:59:59,519 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e57f485022f54d99a2e82c75c1565ace to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/e57f485022f54d99a2e82c75c1565ace 2024-12-04T14:59:59,521 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/3821ef0d222647cf8e64cf8f44606b4b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/d175a81cfc8f44529c1eb682adb4e370, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/7ba4de2ffaba4f3696d9dc22b470db19, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/fad1482a72594bbc82bd06e9d356587c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/29516cc4fda64541aa1dd5c17b69e5cd, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/24a2c7115df9470eb475ce68177e364a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/df7ebbde767e420e89d12c38ea31257f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/ba63c803d96041279b30f15c11a6345f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/c920507d10ef41f3842d8ceb0ffd7573, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/16b281418e2348d7bbe0af8d188e04d0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/d35f685c6ea14fe6bf32493616032ffb, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/c1450696ef0f4716b30bb992c2a16f37, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/0926ed29787f463dbf19a6036708c134, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/643d77e75a794a18ae24ca8e1dded3f9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/27650e44c53a444f95c678490479be24, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/5794a80b6a1a4605a33e554d2e1d9e68, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/3eed1159ac7243ea8ff7718c28d0084c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/1d8aac34ea2c4afaaa93f522cb806a4c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/0e62acb5ccff4fd99ee62a9b5e88035c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/39a1df2348464d34a9cacf3e094d2e72, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/9b1a25bc4f8e4d03968bd7f442230667, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/e66d14b5f3604e3587978a93fde67755, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/93503448702349d4b7412dbfef57815a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/1ada3fc975d941e395821c2c26cf84be, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/40a245aa49af421c8fc83377c8ed7668, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/b6a0c6715f9b4322b5886da634f7b17e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/820b85e367354480acf5e67e31335870, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/92a2961f320f458f89c1d6dd3f120fb2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/99d2f911cb194f24b36c1e953c981410, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/4e470fe51a934bcbb5d91173e81c7771, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/00eb91da927944b38542c9d7852abfbe] to archive 2024-12-04T14:59:59,521 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:59:59,523 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/3821ef0d222647cf8e64cf8f44606b4b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/3821ef0d222647cf8e64cf8f44606b4b 2024-12-04T14:59:59,523 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/d175a81cfc8f44529c1eb682adb4e370 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/d175a81cfc8f44529c1eb682adb4e370 2024-12-04T14:59:59,524 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/7ba4de2ffaba4f3696d9dc22b470db19 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/7ba4de2ffaba4f3696d9dc22b470db19 2024-12-04T14:59:59,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/fad1482a72594bbc82bd06e9d356587c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/fad1482a72594bbc82bd06e9d356587c 2024-12-04T14:59:59,526 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/29516cc4fda64541aa1dd5c17b69e5cd to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/29516cc4fda64541aa1dd5c17b69e5cd 2024-12-04T14:59:59,527 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/24a2c7115df9470eb475ce68177e364a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/24a2c7115df9470eb475ce68177e364a 2024-12-04T14:59:59,528 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/df7ebbde767e420e89d12c38ea31257f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/df7ebbde767e420e89d12c38ea31257f 2024-12-04T14:59:59,528 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/ba63c803d96041279b30f15c11a6345f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/ba63c803d96041279b30f15c11a6345f 2024-12-04T14:59:59,529 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/c920507d10ef41f3842d8ceb0ffd7573 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/c920507d10ef41f3842d8ceb0ffd7573 2024-12-04T14:59:59,530 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/16b281418e2348d7bbe0af8d188e04d0 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/16b281418e2348d7bbe0af8d188e04d0 2024-12-04T14:59:59,530 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/d35f685c6ea14fe6bf32493616032ffb to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/d35f685c6ea14fe6bf32493616032ffb 2024-12-04T14:59:59,531 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/c1450696ef0f4716b30bb992c2a16f37 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/c1450696ef0f4716b30bb992c2a16f37 2024-12-04T14:59:59,531 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/0926ed29787f463dbf19a6036708c134 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/0926ed29787f463dbf19a6036708c134 2024-12-04T14:59:59,532 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/643d77e75a794a18ae24ca8e1dded3f9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/643d77e75a794a18ae24ca8e1dded3f9 2024-12-04T14:59:59,533 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/27650e44c53a444f95c678490479be24 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/27650e44c53a444f95c678490479be24 2024-12-04T14:59:59,533 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/5794a80b6a1a4605a33e554d2e1d9e68 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/5794a80b6a1a4605a33e554d2e1d9e68 2024-12-04T14:59:59,534 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/3eed1159ac7243ea8ff7718c28d0084c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/3eed1159ac7243ea8ff7718c28d0084c 2024-12-04T14:59:59,535 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/1d8aac34ea2c4afaaa93f522cb806a4c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/1d8aac34ea2c4afaaa93f522cb806a4c 2024-12-04T14:59:59,535 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/0e62acb5ccff4fd99ee62a9b5e88035c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/0e62acb5ccff4fd99ee62a9b5e88035c 2024-12-04T14:59:59,536 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/39a1df2348464d34a9cacf3e094d2e72 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/39a1df2348464d34a9cacf3e094d2e72 2024-12-04T14:59:59,537 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/9b1a25bc4f8e4d03968bd7f442230667 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/9b1a25bc4f8e4d03968bd7f442230667 2024-12-04T14:59:59,537 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/e66d14b5f3604e3587978a93fde67755 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/e66d14b5f3604e3587978a93fde67755 2024-12-04T14:59:59,538 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/93503448702349d4b7412dbfef57815a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/93503448702349d4b7412dbfef57815a 2024-12-04T14:59:59,539 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/1ada3fc975d941e395821c2c26cf84be to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/1ada3fc975d941e395821c2c26cf84be 2024-12-04T14:59:59,540 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/40a245aa49af421c8fc83377c8ed7668 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/40a245aa49af421c8fc83377c8ed7668 2024-12-04T14:59:59,540 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/b6a0c6715f9b4322b5886da634f7b17e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/b6a0c6715f9b4322b5886da634f7b17e 2024-12-04T14:59:59,541 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/820b85e367354480acf5e67e31335870 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/820b85e367354480acf5e67e31335870 2024-12-04T14:59:59,542 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/92a2961f320f458f89c1d6dd3f120fb2 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/92a2961f320f458f89c1d6dd3f120fb2 2024-12-04T14:59:59,542 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/99d2f911cb194f24b36c1e953c981410 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/99d2f911cb194f24b36c1e953c981410 2024-12-04T14:59:59,543 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/4e470fe51a934bcbb5d91173e81c7771 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/4e470fe51a934bcbb5d91173e81c7771 2024-12-04T14:59:59,544 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/00eb91da927944b38542c9d7852abfbe to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/00eb91da927944b38542c9d7852abfbe 2024-12-04T14:59:59,545 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/306f1512844d4026aaf0cb8688516868, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/464c6bab6cad4169945f35d6b066d622, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/0809e007e136429699aca611d3c11c90, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/db5409623fdc4f898869da96a6314101, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/150463d083ba45cd990cdd35d84350a3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/13e84ff1f3914664a042977ddcc76d9f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/946b04b9318f484383c706d77af2f452, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/efd2f68a03494af087ca10d81d67dfc6, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/340244ad88b14dae893e676157547927, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/f50464766cc549ebba761e92ba618d09, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/a1665cbfac0f4bc7abc35c3c59370330, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/e794136acdc448bf96864e631563675d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/2cd3b45b1d084c04b999f19f38e66307, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/432fd6483b6b47d19d67e15f096a3030, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/a298f9e38e964c1398fdda4ad31dad98, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/d79e05ca0fbe467b8ef17bab8387dbb3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/7e15e6e91c4b46eba71aafdddb38e305, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/b20f8a7e58ff4808a3ba24583d96fdd6, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/6b9483e57a214568a06e5dd340a2284b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/aad2883f41974144adb303ee279587b2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/979913b7729e4f7aa2525293c74eed4b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/a12cad326ccf4f66b6811cecbb3a45f0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/bce6de973756473cb9e03e8e3699b5e9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/54085a1ebe5f48dda7627fd51521d60f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/630db658626a4eb680c314ae4622bbf3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/7b2a0607f85a4823a6610884466173b5, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/9de0771ab6444c77bfc6b95bbe246b2b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/f5f021830b484a278655b5e2554d3168, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/0eaef9e75e8d484b916f534e64a99993, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/d9d883b7645046b2827b03c8b5739ade, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/486ba5ea42f644b89dc0d8ce768c6c18] to archive 2024-12-04T14:59:59,545 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:59:59,546 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/306f1512844d4026aaf0cb8688516868 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/306f1512844d4026aaf0cb8688516868 2024-12-04T14:59:59,547 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/464c6bab6cad4169945f35d6b066d622 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/464c6bab6cad4169945f35d6b066d622 2024-12-04T14:59:59,548 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/0809e007e136429699aca611d3c11c90 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/0809e007e136429699aca611d3c11c90 2024-12-04T14:59:59,549 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/db5409623fdc4f898869da96a6314101 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/db5409623fdc4f898869da96a6314101 2024-12-04T14:59:59,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/150463d083ba45cd990cdd35d84350a3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/150463d083ba45cd990cdd35d84350a3 2024-12-04T14:59:59,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/13e84ff1f3914664a042977ddcc76d9f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/13e84ff1f3914664a042977ddcc76d9f 2024-12-04T14:59:59,551 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/946b04b9318f484383c706d77af2f452 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/946b04b9318f484383c706d77af2f452 2024-12-04T14:59:59,552 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/efd2f68a03494af087ca10d81d67dfc6 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/efd2f68a03494af087ca10d81d67dfc6 2024-12-04T14:59:59,553 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/340244ad88b14dae893e676157547927 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/340244ad88b14dae893e676157547927 2024-12-04T14:59:59,554 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/f50464766cc549ebba761e92ba618d09 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/f50464766cc549ebba761e92ba618d09 2024-12-04T14:59:59,554 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/a1665cbfac0f4bc7abc35c3c59370330 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/a1665cbfac0f4bc7abc35c3c59370330 2024-12-04T14:59:59,555 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/e794136acdc448bf96864e631563675d to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/e794136acdc448bf96864e631563675d 2024-12-04T14:59:59,556 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/2cd3b45b1d084c04b999f19f38e66307 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/2cd3b45b1d084c04b999f19f38e66307 2024-12-04T14:59:59,557 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/432fd6483b6b47d19d67e15f096a3030 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/432fd6483b6b47d19d67e15f096a3030 2024-12-04T14:59:59,557 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/a298f9e38e964c1398fdda4ad31dad98 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/a298f9e38e964c1398fdda4ad31dad98 2024-12-04T14:59:59,558 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/d79e05ca0fbe467b8ef17bab8387dbb3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/d79e05ca0fbe467b8ef17bab8387dbb3 2024-12-04T14:59:59,559 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/7e15e6e91c4b46eba71aafdddb38e305 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/7e15e6e91c4b46eba71aafdddb38e305 2024-12-04T14:59:59,560 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/b20f8a7e58ff4808a3ba24583d96fdd6 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/b20f8a7e58ff4808a3ba24583d96fdd6 2024-12-04T14:59:59,560 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/6b9483e57a214568a06e5dd340a2284b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/6b9483e57a214568a06e5dd340a2284b 2024-12-04T14:59:59,561 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/aad2883f41974144adb303ee279587b2 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/aad2883f41974144adb303ee279587b2 2024-12-04T14:59:59,562 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/979913b7729e4f7aa2525293c74eed4b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/979913b7729e4f7aa2525293c74eed4b 2024-12-04T14:59:59,563 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/a12cad326ccf4f66b6811cecbb3a45f0 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/a12cad326ccf4f66b6811cecbb3a45f0 2024-12-04T14:59:59,564 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/bce6de973756473cb9e03e8e3699b5e9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/bce6de973756473cb9e03e8e3699b5e9 2024-12-04T14:59:59,565 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/54085a1ebe5f48dda7627fd51521d60f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/54085a1ebe5f48dda7627fd51521d60f 2024-12-04T14:59:59,566 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/630db658626a4eb680c314ae4622bbf3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/630db658626a4eb680c314ae4622bbf3 2024-12-04T14:59:59,567 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/7b2a0607f85a4823a6610884466173b5 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/7b2a0607f85a4823a6610884466173b5 2024-12-04T14:59:59,567 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/9de0771ab6444c77bfc6b95bbe246b2b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/9de0771ab6444c77bfc6b95bbe246b2b 2024-12-04T14:59:59,568 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/f5f021830b484a278655b5e2554d3168 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/f5f021830b484a278655b5e2554d3168 2024-12-04T14:59:59,569 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/0eaef9e75e8d484b916f534e64a99993 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/0eaef9e75e8d484b916f534e64a99993 2024-12-04T14:59:59,570 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/d9d883b7645046b2827b03c8b5739ade to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/d9d883b7645046b2827b03c8b5739ade 2024-12-04T14:59:59,570 DEBUG [StoreCloser-TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/486ba5ea42f644b89dc0d8ce768c6c18 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/486ba5ea42f644b89dc0d8ce768c6c18 2024-12-04T14:59:59,574 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/recovered.edits/556.seqid, newMaxSeqId=556, maxSeqId=1 2024-12-04T14:59:59,574 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. 2024-12-04T14:59:59,574 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1676): Region close journal for cf9bed3c8089ea388e8abd4912312aee: Waiting for close lock at 1733324398238Running coprocessor pre-close hooks at 1733324398238Disabling compacts and flushes for region at 1733324398238Disabling writes for close at 1733324398238Obtaining lock to block concurrent updates at 1733324398239 (+1 ms)Preparing flush snapshotting stores in cf9bed3c8089ea388e8abd4912312aee at 1733324398239Finished memstore snapshotting TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733324398239Flushing stores of TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee. at 1733324398240 (+1 ms)Flushing cf9bed3c8089ea388e8abd4912312aee/A: creating writer at 1733324398240Flushing cf9bed3c8089ea388e8abd4912312aee/A: appending metadata at 1733324398242 (+2 ms)Flushing cf9bed3c8089ea388e8abd4912312aee/A: closing flushed file at 1733324398242Flushing cf9bed3c8089ea388e8abd4912312aee/B: creating writer at 1733324398652 (+410 ms)Flushing cf9bed3c8089ea388e8abd4912312aee/B: appending metadata at 1733324398655 (+3 ms)Flushing cf9bed3c8089ea388e8abd4912312aee/B: closing flushed file at 1733324398655Flushing cf9bed3c8089ea388e8abd4912312aee/C: creating writer at 1733324399064 (+409 ms)Flushing cf9bed3c8089ea388e8abd4912312aee/C: appending metadata at 1733324399067 (+3 ms)Flushing cf9bed3c8089ea388e8abd4912312aee/C: closing flushed file at 1733324399067Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c813cb: reopening flushed file at 1733324399476 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32a07308: reopening flushed file at 1733324399480 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dc93628: reopening flushed file at 1733324399485 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for cf9bed3c8089ea388e8abd4912312aee in 1253ms, sequenceid=553, compaction requested=true at 1733324399491 (+6 ms)Writing region close event to WAL at 1733324399571 (+80 ms)Running coprocessor post-close hooks at 1733324399574 (+3 ms)Closed at 1733324399574 2024-12-04T14:59:59,576 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(157): Closed cf9bed3c8089ea388e8abd4912312aee 2024-12-04T14:59:59,576 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=cf9bed3c8089ea388e8abd4912312aee, regionState=CLOSED 2024-12-04T14:59:59,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 because future has completed 2024-12-04T14:59:59,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-04T14:59:59,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseRegionProcedure cf9bed3c8089ea388e8abd4912312aee, server=d214edae7bd2,36649,1733324304801 in 1.4930 sec 2024-12-04T14:59:59,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-12-04T14:59:59,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf9bed3c8089ea388e8abd4912312aee, UNASSIGN in 1.4980 sec 2024-12-04T14:59:59,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-04T14:59:59,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5010 sec 2024-12-04T14:59:59,583 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324399582"}]},"ts":"1733324399582"} 2024-12-04T14:59:59,584 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T14:59:59,584 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T14:59:59,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5100 sec 2024-12-04T15:00:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T15:00:00,220 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:00,221 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T15:00:00,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:00:00,222 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:00:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T15:00:00,223 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=80, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:00:00,225 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee 2024-12-04T15:00:00,227 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A, FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B, FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C, FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/recovered.edits] 2024-12-04T15:00:00,228 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/0510541118df4d22b3e127418e55f749 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/0510541118df4d22b3e127418e55f749 2024-12-04T15:00:00,229 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/5e3fc5f26650434c823fa5af312738e3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/5e3fc5f26650434c823fa5af312738e3 2024-12-04T15:00:00,230 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/fc9752e581e845498d510397fcf5ff26 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/A/fc9752e581e845498d510397fcf5ff26 2024-12-04T15:00:00,233 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/1c08b65c68bc491682a3a61d8a1790fa to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/1c08b65c68bc491682a3a61d8a1790fa 2024-12-04T15:00:00,234 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/2d27f68618154a68a5a890f3cfecae56 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/2d27f68618154a68a5a890f3cfecae56 2024-12-04T15:00:00,235 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/e93f64eea4ef4ff3b6daf14d35a2ac57 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/B/e93f64eea4ef4ff3b6daf14d35a2ac57 2024-12-04T15:00:00,237 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/36f8af5c462c48f786f44e09a8fa2aa0 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/36f8af5c462c48f786f44e09a8fa2aa0 2024-12-04T15:00:00,238 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/7b8d76087b4c4e5da0d3a3f8d8955cfd to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/7b8d76087b4c4e5da0d3a3f8d8955cfd 2024-12-04T15:00:00,240 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/c848679f232149b09859625c09cbe329 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/C/c848679f232149b09859625c09cbe329 2024-12-04T15:00:00,242 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/recovered.edits/556.seqid to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee/recovered.edits/556.seqid 2024-12-04T15:00:00,243 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/cf9bed3c8089ea388e8abd4912312aee 2024-12-04T15:00:00,243 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T15:00:00,245 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=80, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:00:00,248 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T15:00:00,250 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T15:00:00,251 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=80, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:00:00,251 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T15:00:00,251 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733324400251"}]},"ts":"9223372036854775807"} 2024-12-04T15:00:00,253 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T15:00:00,253 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cf9bed3c8089ea388e8abd4912312aee, NAME => 'TestAcidGuarantees,,1733324373217.cf9bed3c8089ea388e8abd4912312aee.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T15:00:00,253 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T15:00:00,254 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733324400253"}]},"ts":"9223372036854775807"} 2024-12-04T15:00:00,256 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T15:00:00,256 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=80, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:00:00,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 35 msec 2024-12-04T15:00:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T15:00:00,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T15:00:00,331 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:00,341 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-325510507_22 at /127.0.0.1:46894 [Waiting for operation #426] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1045188705_22 at /127.0.0.1:50324 [Waiting for operation #202] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=532 (was 597), ProcessCount=11 (was 11), AvailableMemoryMB=9795 (was 9265) - AvailableMemoryMB LEAK? - 2024-12-04T15:00:00,349 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=532, ProcessCount=11, AvailableMemoryMB=9795 2024-12-04T15:00:00,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T15:00:00,350 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:00:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T15:00:00,352 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T15:00:00,352 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:00,353 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T15:00:00,353 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 81 2024-12-04T15:00:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-04T15:00:00,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742154_1330 (size=1061) 2024-12-04T15:00:00,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-04T15:00:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-04T15:00:00,760 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee 2024-12-04T15:00:00,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742155_1331 (size=53) 2024-12-04T15:00:00,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-04T15:00:01,165 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:00:01,165 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e03f7dfff5f07d53203dd271e7128a0a, disabling compactions & flushes 2024-12-04T15:00:01,165 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:01,165 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:01,165 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. after waiting 0 ms 2024-12-04T15:00:01,166 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:01,166 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:01,166 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e03f7dfff5f07d53203dd271e7128a0a: Waiting for close lock at 1733324401165Disabling compacts and flushes for region at 1733324401165Disabling writes for close at 1733324401166 (+1 ms)Writing region close event to WAL at 1733324401166Closed at 1733324401166 2024-12-04T15:00:01,167 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T15:00:01,167 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733324401167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733324401167"}]},"ts":"1733324401167"} 2024-12-04T15:00:01,168 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T15:00:01,169 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T15:00:01,169 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324401169"}]},"ts":"1733324401169"} 2024-12-04T15:00:01,171 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T15:00:01,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e03f7dfff5f07d53203dd271e7128a0a, ASSIGN}] 2024-12-04T15:00:01,172 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e03f7dfff5f07d53203dd271e7128a0a, ASSIGN 2024-12-04T15:00:01,173 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e03f7dfff5f07d53203dd271e7128a0a, ASSIGN; state=OFFLINE, location=d214edae7bd2,36649,1733324304801; forceNewPlan=false, retain=false 2024-12-04T15:00:01,323 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=e03f7dfff5f07d53203dd271e7128a0a, regionState=OPENING, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:01,325 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e03f7dfff5f07d53203dd271e7128a0a, ASSIGN because future has completed 2024-12-04T15:00:01,326 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801}] 2024-12-04T15:00:01,480 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:01,480 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7752): Opening region: {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} 2024-12-04T15:00:01,481 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:01,481 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:00:01,481 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7794): checking encryption for e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:01,481 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7797): checking classloading for e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:01,482 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:01,483 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T15:00:01,484 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e03f7dfff5f07d53203dd271e7128a0a columnFamilyName A 2024-12-04T15:00:01,484 DEBUG [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:01,484 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] regionserver.HStore(327): Store=e03f7dfff5f07d53203dd271e7128a0a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:00:01,484 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:01,485 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T15:00:01,485 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e03f7dfff5f07d53203dd271e7128a0a columnFamilyName B 2024-12-04T15:00:01,485 DEBUG [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:01,486 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] regionserver.HStore(327): Store=e03f7dfff5f07d53203dd271e7128a0a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:00:01,486 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:01,487 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T15:00:01,487 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e03f7dfff5f07d53203dd271e7128a0a columnFamilyName C 2024-12-04T15:00:01,487 DEBUG [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:01,488 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] regionserver.HStore(327): Store=e03f7dfff5f07d53203dd271e7128a0a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:00:01,488 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:01,488 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1038): replaying wal for e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:01,489 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:01,489 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:01,489 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1048): stopping wal replay for e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:01,489 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1060): Cleaning up temporary data for e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:01,490 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T15:00:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-04T15:00:01,491 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1093): writing seq id for e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:01,492 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T15:00:01,493 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1114): Opened e03f7dfff5f07d53203dd271e7128a0a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62340236, jitterRate=-0.07105809450149536}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T15:00:01,493 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:01,493 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1006): Region open journal for e03f7dfff5f07d53203dd271e7128a0a: Running coprocessor pre-open hook at 1733324401481Writing region info on filesystem at 1733324401481Initializing all the Stores at 1733324401482 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324401482Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324401482Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324401482Cleaning up temporary data from old regions at 1733324401489 (+7 ms)Running coprocessor post-open hooks at 1733324401493 (+4 ms)Region opened successfully at 1733324401493 2024-12-04T15:00:01,494 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., pid=83, masterSystemTime=1733324401478 2024-12-04T15:00:01,496 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:01,496 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:01,496 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=e03f7dfff5f07d53203dd271e7128a0a, regionState=OPEN, openSeqNum=2, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:01,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 because future has completed 2024-12-04T15:00:01,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-04T15:00:01,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 in 172 msec 2024-12-04T15:00:01,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-04T15:00:01,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e03f7dfff5f07d53203dd271e7128a0a, ASSIGN in 329 msec 2024-12-04T15:00:01,503 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T15:00:01,503 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324401503"}]},"ts":"1733324401503"} 2024-12-04T15:00:01,504 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T15:00:01,505 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T15:00:01,506 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1550 sec 2024-12-04T15:00:02,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-04T15:00:02,501 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:02,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@211104f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:02,502 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:02,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:02,503 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:02,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:02,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:02,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f8943f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:02,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:02,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:02,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:02,504 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:02,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6197b0a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:02,505 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:02,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:02,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:02,506 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:02,507 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T15:00:02,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is d214edae7bd2,39449,1733324304115 2024-12-04T15:00:02,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4a8d12e7 2024-12-04T15:00:02,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T15:00:02,508 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T15:00:02,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T15:00:02,510 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:00:02,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-04T15:00:02,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742156_1332 (size=1097) 2024-12-04T15:00:02,920 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-04T15:00:02,920 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-04T15:00:02,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T15:00:02,925 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e03f7dfff5f07d53203dd271e7128a0a, REOPEN/MOVE}] 2024-12-04T15:00:02,926 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e03f7dfff5f07d53203dd271e7128a0a, REOPEN/MOVE 2024-12-04T15:00:02,927 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=e03f7dfff5f07d53203dd271e7128a0a, regionState=CLOSING, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:02,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e03f7dfff5f07d53203dd271e7128a0a, REOPEN/MOVE because future has completed 2024-12-04T15:00:02,928 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T15:00:02,928 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801}] 2024-12-04T15:00:03,081 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:03,081 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T15:00:03,081 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing e03f7dfff5f07d53203dd271e7128a0a, disabling compactions & flushes 2024-12-04T15:00:03,081 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:03,081 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:03,081 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. after waiting 0 ms 2024-12-04T15:00:03,081 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:03,085 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-04T15:00:03,085 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:03,085 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for e03f7dfff5f07d53203dd271e7128a0a: Waiting for close lock at 1733324403081Running coprocessor pre-close hooks at 1733324403081Disabling compacts and flushes for region at 1733324403081Disabling writes for close at 1733324403081Writing region close event to WAL at 1733324403081Running coprocessor post-close hooks at 1733324403085 (+4 ms)Closed at 1733324403085 2024-12-04T15:00:03,085 WARN [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionServer(3299): Not adding moved region record: e03f7dfff5f07d53203dd271e7128a0a to self. 2024-12-04T15:00:03,087 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:03,087 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=e03f7dfff5f07d53203dd271e7128a0a, regionState=CLOSED 2024-12-04T15:00:03,089 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 because future has completed 2024-12-04T15:00:03,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-04T15:00:03,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 in 161 msec 2024-12-04T15:00:03,092 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e03f7dfff5f07d53203dd271e7128a0a, REOPEN/MOVE; state=CLOSED, location=d214edae7bd2,36649,1733324304801; forceNewPlan=false, retain=true 2024-12-04T15:00:03,242 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=e03f7dfff5f07d53203dd271e7128a0a, regionState=OPENING, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:03,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e03f7dfff5f07d53203dd271e7128a0a, REOPEN/MOVE because future has completed 2024-12-04T15:00:03,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801}] 2024-12-04T15:00:03,398 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:03,398 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7752): Opening region: {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} 2024-12-04T15:00:03,398 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:03,399 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:00:03,399 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7794): checking encryption for e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:03,399 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7797): checking classloading for e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:03,400 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:03,401 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T15:00:03,401 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e03f7dfff5f07d53203dd271e7128a0a columnFamilyName A 2024-12-04T15:00:03,402 DEBUG [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:03,402 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] regionserver.HStore(327): Store=e03f7dfff5f07d53203dd271e7128a0a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:00:03,402 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:03,403 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T15:00:03,403 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e03f7dfff5f07d53203dd271e7128a0a columnFamilyName B 2024-12-04T15:00:03,403 DEBUG [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:03,403 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] regionserver.HStore(327): Store=e03f7dfff5f07d53203dd271e7128a0a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:00:03,403 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:03,404 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T15:00:03,404 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e03f7dfff5f07d53203dd271e7128a0a columnFamilyName C 2024-12-04T15:00:03,404 DEBUG [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:03,404 INFO [StoreOpener-e03f7dfff5f07d53203dd271e7128a0a-1 {}] regionserver.HStore(327): Store=e03f7dfff5f07d53203dd271e7128a0a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:00:03,404 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:03,404 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1038): replaying wal for e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:03,405 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:03,406 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:03,407 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1048): stopping wal replay for e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:03,407 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1060): Cleaning up temporary data for e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:03,407 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T15:00:03,408 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): writing seq id for e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:03,409 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1114): Opened e03f7dfff5f07d53203dd271e7128a0a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65715325, jitterRate=-0.02076534926891327}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T15:00:03,409 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:03,409 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1006): Region open journal for e03f7dfff5f07d53203dd271e7128a0a: Running coprocessor pre-open hook at 1733324403399Writing region info on filesystem at 1733324403399Initializing all the Stores at 1733324403399Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324403399Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324403399Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324403399Cleaning up temporary data from old regions at 1733324403407 (+8 ms)Running coprocessor post-open hooks at 1733324403409 (+2 ms)Region opened successfully at 1733324403409 2024-12-04T15:00:03,410 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., pid=88, masterSystemTime=1733324403395 2024-12-04T15:00:03,411 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:03,411 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:03,412 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=e03f7dfff5f07d53203dd271e7128a0a, regionState=OPEN, openSeqNum=5, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:03,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 because future has completed 2024-12-04T15:00:03,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=86 2024-12-04T15:00:03,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 in 169 msec 2024-12-04T15:00:03,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-04T15:00:03,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e03f7dfff5f07d53203dd271e7128a0a, REOPEN/MOVE in 490 msec 2024-12-04T15:00:03,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-04T15:00:03,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 493 msec 2024-12-04T15:00:03,420 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 908 msec 2024-12-04T15:00:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-04T15:00:03,421 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:03,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c06627f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,421 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:03,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:03,422 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:03,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d886860, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:03,424 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c6ceef0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8673cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,425 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:03,425 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:03,426 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b651e96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:03,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:03,427 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:03,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a6f5c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18ca1e42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,428 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:03,428 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:03,429 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:03,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:03,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:03,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c9925ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:03,430 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fc0499f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c7eae17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,432 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:03,432 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:03,432 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:03,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:03,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:03,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cfa378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:03,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:03,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:03,433 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:03,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ffd1f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73c5be4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,435 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:03,435 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:03,435 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae22e3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:03,436 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2616f09f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@395008aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:03,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:03,438 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:03,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@620ee92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:03,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:03,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:03,439 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d69250, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61351bc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,441 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:03,441 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:03,441 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:03,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:03,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:03,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c4bbd71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:03,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:03,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:03,442 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@273440bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6444e4e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,443 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:03,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:03,444 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f7c75e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:03,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:03,445 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5739e6ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b350ef2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,446 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:03,446 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:03,447 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:03,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:03,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a996506, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:03,449 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78afa929, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775a762e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,450 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:03,450 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:03,451 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:03,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2618d75d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:03,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:03,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:03,453 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:03,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bec94fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:03,455 DEBUG [Thread-1525 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:03,455 DEBUG [Thread-1527 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:03,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:03,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:03,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:03,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:03,457 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:03,457 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:03,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T15:00:03,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T15:00:03,465 DEBUG [Thread-1519 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:03,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:03,466 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T15:00:03,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:03,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:03,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:03,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:03,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:03,466 DEBUG [Thread-1532 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:03,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:03,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:03,467 DEBUG [Thread-1521 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:03,467 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:03,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:03,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:03,469 DEBUG [Thread-1536 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:03,471 DEBUG [Thread-1536 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:03,472 DEBUG [Thread-1536 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T15:00:03,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:03,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:03,474 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:03,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T15:00:03,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T15:00:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T15:00:03,490 DEBUG [Thread-1530 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:03,491 DEBUG [Thread-1523 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:03,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:03,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:03,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:03,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:03,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:03,496 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T15:00:03,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733324403479, max timestamp=1733324403494Num uniques 859; 2024-12-04T15:00:03,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733324403479, max timestamp=1733324403494Num uniques 859; 2024-12-04T15:00:03,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T15:00:03,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T15:00:03,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T15:00:03,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733324403479, max timestamp=1733324403494Num uniques 859; 2024-12-04T15:00:03,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T15:00:03,507 DEBUG [Thread-1534 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:03,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:03,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:03,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412046886ea7ff5914f4783d72d1978b59158_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324403465/Put/seqid=0 2024-12-04T15:00:03,510 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:03,511 DEBUG [Thread-1538 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:03,515 DEBUG [Thread-1538 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:03,515 DEBUG [Thread-1538 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:03,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324463511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:03,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324463512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:03,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:00:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-12-04T15:00:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-04T15:00:03,518 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:00:03,519 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:00:03,519 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:00:03,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:03,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324463515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:03,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324463516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:03,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:03,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324463520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:03,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T15:00:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,531 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:03,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T15:00:03,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742157_1333 (size=12154) 2024-12-04T15:00:03,549 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:03,553 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412046886ea7ff5914f4783d72d1978b59158_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046886ea7ff5914f4783d72d1978b59158_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:03,558 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/59e4b30bb40847f7a11c7c316bbac777, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:03,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/59e4b30bb40847f7a11c7c316bbac777 is 175, key is test_row_0/A:col10/1733324403465/Put/seqid=0 2024-12-04T15:00:03,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742158_1334 (size=30955) 2024-12-04T15:00:03,571 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/59e4b30bb40847f7a11c7c316bbac777 2024-12-04T15:00:03,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/c5323683d3d24a319a119689c08e4a25 is 50, key is test_row_0/B:col10/1733324403465/Put/seqid=0 2024-12-04T15:00:03,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742159_1335 (size=12001) 2024-12-04T15:00:03,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/c5323683d3d24a319a119689c08e4a25 2024-12-04T15:00:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-04T15:00:03,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/6ed42f7bbfed4f978f19d7469efe74d8 is 50, key is test_row_0/C:col10/1733324403465/Put/seqid=0 2024-12-04T15:00:03,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324463630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:03,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324463630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:03,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324463631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:03,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:03,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:03,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324463640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:03,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:03,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324463641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:03,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:03,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742160_1336 (size=12001) 2024-12-04T15:00:03,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-04T15:00:03,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:03,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:03,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:03,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:03,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:03,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-04T15:00:03,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:03,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:03,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:03,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:03,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-04T15:00:03,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:03,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324463840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:03,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:03,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:03,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324463840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:03,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:03,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324463840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:03,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:03,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:03,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324463860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:03,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324463860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:03,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-04T15:00:03,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:03,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:03,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:03,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:03,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:03,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:04,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/6ed42f7bbfed4f978f19d7469efe74d8 2024-12-04T15:00:04,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/59e4b30bb40847f7a11c7c316bbac777 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/59e4b30bb40847f7a11c7c316bbac777 2024-12-04T15:00:04,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/59e4b30bb40847f7a11c7c316bbac777, entries=150, sequenceid=15, filesize=30.2 K 2024-12-04T15:00:04,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/c5323683d3d24a319a119689c08e4a25 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c5323683d3d24a319a119689c08e4a25 2024-12-04T15:00:04,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c5323683d3d24a319a119689c08e4a25, entries=150, sequenceid=15, filesize=11.7 K 2024-12-04T15:00:04,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/6ed42f7bbfed4f978f19d7469efe74d8 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/6ed42f7bbfed4f978f19d7469efe74d8 2024-12-04T15:00:04,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/6ed42f7bbfed4f978f19d7469efe74d8, entries=150, sequenceid=15, filesize=11.7 K 2024-12-04T15:00:04,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for e03f7dfff5f07d53203dd271e7128a0a in 631ms, sequenceid=15, compaction requested=false 2024-12-04T15:00:04,097 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-04T15:00:04,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:04,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-04T15:00:04,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:04,134 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-12-04T15:00:04,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:04,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:04,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:04,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:04,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:04,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-04T15:00:04,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:04,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:04,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a363828b52d74cf1841bcb9f04d0b7ec_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324403510/Put/seqid=0 2024-12-04T15:00:04,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324464186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324464187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742161_1337 (size=12154) 2024-12-04T15:00:04,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:04,212 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a363828b52d74cf1841bcb9f04d0b7ec_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a363828b52d74cf1841bcb9f04d0b7ec_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:04,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/c5301bf669e54845837140cf12128c3e, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:04,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324464190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/c5301bf669e54845837140cf12128c3e is 175, key is test_row_0/A:col10/1733324403510/Put/seqid=0 2024-12-04T15:00:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324464197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324464202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742162_1338 (size=30955) 2024-12-04T15:00:04,257 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/c5301bf669e54845837140cf12128c3e 2024-12-04T15:00:04,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/099360fef94247859824b36e05b277e5 is 50, key is test_row_0/B:col10/1733324403510/Put/seqid=0 2024-12-04T15:00:04,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742163_1339 (size=12001) 2024-12-04T15:00:04,309 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/099360fef94247859824b36e05b277e5 2024-12-04T15:00:04,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324464310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324464312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324464321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324464321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324464321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/51530c3cc9f744ddb614f31d5f58d07f is 50, key is test_row_0/C:col10/1733324403510/Put/seqid=0 2024-12-04T15:00:04,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742164_1340 (size=12001) 2024-12-04T15:00:04,368 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/51530c3cc9f744ddb614f31d5f58d07f 2024-12-04T15:00:04,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/c5301bf669e54845837140cf12128c3e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/c5301bf669e54845837140cf12128c3e 2024-12-04T15:00:04,387 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/c5301bf669e54845837140cf12128c3e, entries=150, sequenceid=49, filesize=30.2 K 2024-12-04T15:00:04,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/099360fef94247859824b36e05b277e5 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/099360fef94247859824b36e05b277e5 2024-12-04T15:00:04,397 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/099360fef94247859824b36e05b277e5, entries=150, sequenceid=49, filesize=11.7 K 2024-12-04T15:00:04,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/51530c3cc9f744ddb614f31d5f58d07f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/51530c3cc9f744ddb614f31d5f58d07f 2024-12-04T15:00:04,406 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/51530c3cc9f744ddb614f31d5f58d07f, entries=150, sequenceid=49, filesize=11.7 K 2024-12-04T15:00:04,407 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for e03f7dfff5f07d53203dd271e7128a0a in 273ms, sequenceid=49, compaction requested=false 2024-12-04T15:00:04,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:04,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:04,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-12-04T15:00:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-12-04T15:00:04,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-04T15:00:04,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 893 msec 2024-12-04T15:00:04,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 899 msec 2024-12-04T15:00:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:04,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T15:00:04,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:04,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:04,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:04,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:04,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:04,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:04,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204aef6328212bc4f369c50a99c90356234_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324404521/Put/seqid=0 2024-12-04T15:00:04,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742165_1341 (size=14594) 2024-12-04T15:00:04,577 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:04,587 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204aef6328212bc4f369c50a99c90356234_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204aef6328212bc4f369c50a99c90356234_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:04,589 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/74eb0e5c11334c34aa3e4b62d9d2e0c4, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:04,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/74eb0e5c11334c34aa3e4b62d9d2e0c4 is 175, key is test_row_0/A:col10/1733324404521/Put/seqid=0 2024-12-04T15:00:04,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324464578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324464580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733324404573, max timestamp=1733324404592Num uniques 859; 2024-12-04T15:00:04,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-04T15:00:04,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733324404573, max timestamp=1733324404592Num uniques 859; 2024-12-04T15:00:04,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-04T15:00:04,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733324404573, max timestamp=1733324404592Num uniques 859; 2024-12-04T15:00:04,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-04T15:00:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742166_1342 (size=39549) 2024-12-04T15:00:04,612 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/74eb0e5c11334c34aa3e4b62d9d2e0c4 2024-12-04T15:00:04,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/c11c881a32284321ad3066a4c5c61e48 is 50, key is test_row_0/B:col10/1733324404521/Put/seqid=0 2024-12-04T15:00:04,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742167_1343 (size=12001) 2024-12-04T15:00:04,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/c11c881a32284321ad3066a4c5c61e48 2024-12-04T15:00:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-04T15:00:04,650 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:04,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/e67fd26e66ff4d96b5f1dcc2c3d66d3a is 50, key is test_row_0/C:col10/1733324404521/Put/seqid=0 2024-12-04T15:00:04,653 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:00:04,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-12-04T15:00:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-04T15:00:04,655 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:00:04,656 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:00:04,656 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:00:04,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742168_1344 (size=12001) 2024-12-04T15:00:04,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/e67fd26e66ff4d96b5f1dcc2c3d66d3a 2024-12-04T15:00:04,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/74eb0e5c11334c34aa3e4b62d9d2e0c4 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/74eb0e5c11334c34aa3e4b62d9d2e0c4 2024-12-04T15:00:04,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/74eb0e5c11334c34aa3e4b62d9d2e0c4, entries=200, sequenceid=61, filesize=38.6 K 2024-12-04T15:00:04,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324464655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/c11c881a32284321ad3066a4c5c61e48 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c11c881a32284321ad3066a4c5c61e48 2024-12-04T15:00:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324464668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c11c881a32284321ad3066a4c5c61e48, entries=150, sequenceid=61, filesize=11.7 K 2024-12-04T15:00:04,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/e67fd26e66ff4d96b5f1dcc2c3d66d3a as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/e67fd26e66ff4d96b5f1dcc2c3d66d3a 2024-12-04T15:00:04,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324464670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/e67fd26e66ff4d96b5f1dcc2c3d66d3a, entries=150, sequenceid=61, filesize=11.7 K 2024-12-04T15:00:04,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=119.58 KB/122445 for e03f7dfff5f07d53203dd271e7128a0a in 153ms, sequenceid=61, compaction requested=true 2024-12-04T15:00:04,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:04,692 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:04,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:00:04,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:04,692 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:04,693 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:04,693 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/A is initiating minor compaction (all files) 2024-12-04T15:00:04,693 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/A in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:04,693 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/59e4b30bb40847f7a11c7c316bbac777, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/c5301bf669e54845837140cf12128c3e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/74eb0e5c11334c34aa3e4b62d9d2e0c4] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=99.1 K 2024-12-04T15:00:04,693 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:04,693 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/59e4b30bb40847f7a11c7c316bbac777, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/c5301bf669e54845837140cf12128c3e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/74eb0e5c11334c34aa3e4b62d9d2e0c4] 2024-12-04T15:00:04,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:00:04,694 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59e4b30bb40847f7a11c7c316bbac777, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733324403463 2024-12-04T15:00:04,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:04,694 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:04,694 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/B is initiating minor compaction (all files) 2024-12-04T15:00:04,694 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/B in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:04,694 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c5323683d3d24a319a119689c08e4a25, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/099360fef94247859824b36e05b277e5, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c11c881a32284321ad3066a4c5c61e48] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=35.2 K 2024-12-04T15:00:04,694 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5301bf669e54845837140cf12128c3e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733324403509 2024-12-04T15:00:04,694 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting c5323683d3d24a319a119689c08e4a25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733324403463 2024-12-04T15:00:04,694 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74eb0e5c11334c34aa3e4b62d9d2e0c4, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733324404183 2024-12-04T15:00:04,695 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 099360fef94247859824b36e05b277e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733324403509 2024-12-04T15:00:04,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:00:04,695 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting c11c881a32284321ad3066a4c5c61e48, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733324404183 2024-12-04T15:00:04,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:04,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=126.28 KB heapSize=362.32 KB 2024-12-04T15:00:04,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:04,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:04,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:04,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:04,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:04,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:04,716 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#B#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:04,717 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/8b2749fc7dba4ef7a1353c99f605f074 is 50, key is test_row_0/B:col10/1733324404521/Put/seqid=0 2024-12-04T15:00:04,718 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:04,743 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412044e16be9a74824ef481c4ca4f073da73a_e03f7dfff5f07d53203dd271e7128a0a store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:04,745 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412044e16be9a74824ef481c4ca4f073da73a_e03f7dfff5f07d53203dd271e7128a0a, store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:04,745 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412044e16be9a74824ef481c4ca4f073da73a_e03f7dfff5f07d53203dd271e7128a0a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:04,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120441537a463ee84bc194c39b5f0f34aea0_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324404706/Put/seqid=0 2024-12-04T15:00:04,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-04T15:00:04,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324464759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324464767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742169_1345 (size=12104) 2024-12-04T15:00:04,802 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/8b2749fc7dba4ef7a1353c99f605f074 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/8b2749fc7dba4ef7a1353c99f605f074 2024-12-04T15:00:04,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324464790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324464791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-04T15:00:04,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:04,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:04,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324464791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:04,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:04,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:04,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,810 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/B of e03f7dfff5f07d53203dd271e7128a0a into 8b2749fc7dba4ef7a1353c99f605f074(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:04,811 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:04,811 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/B, priority=13, startTime=1733324404692; duration=0sec 2024-12-04T15:00:04,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:04,811 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:04,811 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:B 2024-12-04T15:00:04,811 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:04,813 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:04,813 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/C is initiating minor compaction (all files) 2024-12-04T15:00:04,813 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/C in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:04,813 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/6ed42f7bbfed4f978f19d7469efe74d8, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/51530c3cc9f744ddb614f31d5f58d07f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/e67fd26e66ff4d96b5f1dcc2c3d66d3a] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=35.2 K 2024-12-04T15:00:04,814 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ed42f7bbfed4f978f19d7469efe74d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733324403463 2024-12-04T15:00:04,814 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 51530c3cc9f744ddb614f31d5f58d07f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733324403509 2024-12-04T15:00:04,814 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting e67fd26e66ff4d96b5f1dcc2c3d66d3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733324404183 2024-12-04T15:00:04,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742171_1347 (size=14594) 2024-12-04T15:00:04,828 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:04,831 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120441537a463ee84bc194c39b5f0f34aea0_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120441537a463ee84bc194c39b5f0f34aea0_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:04,832 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/44cd52a3414b465694e5949121fa40d0, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:04,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742170_1346 (size=4469) 2024-12-04T15:00:04,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/44cd52a3414b465694e5949121fa40d0 is 175, key is test_row_0/A:col10/1733324404706/Put/seqid=0 2024-12-04T15:00:04,844 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#C#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:04,845 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/644a962ccf5a4718a5dfbf5e454e7936 is 50, key is test_row_0/C:col10/1733324404521/Put/seqid=0 2024-12-04T15:00:04,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742172_1348 (size=39549) 2024-12-04T15:00:04,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742173_1349 (size=12104) 2024-12-04T15:00:04,860 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/644a962ccf5a4718a5dfbf5e454e7936 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/644a962ccf5a4718a5dfbf5e454e7936 2024-12-04T15:00:04,864 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/C of e03f7dfff5f07d53203dd271e7128a0a into 644a962ccf5a4718a5dfbf5e454e7936(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:04,865 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:04,865 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/C, priority=13, startTime=1733324404695; duration=0sec 2024-12-04T15:00:04,865 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:04,865 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:C 2024-12-04T15:00:04,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324464884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:04,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324464890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:04,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:04,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:04,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-04T15:00:04,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:04,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:04,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:04,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:04,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:04,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:04,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-04T15:00:05,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:05,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324465010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:05,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:05,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324465022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:05,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:05,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324465022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:05,093 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T15:00:05,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324465105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:05,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324465110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:05,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-04T15:00:05,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:05,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:05,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:05,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:05,234 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#A#compaction#295 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:05,235 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/e0af19c292c04778995adabf3ec6499b is 175, key is test_row_0/A:col10/1733324404521/Put/seqid=0 2024-12-04T15:00:05,250 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=42.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/44cd52a3414b465694e5949121fa40d0 2024-12-04T15:00:05,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-04T15:00:05,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:05,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:05,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:05,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:05,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:05,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742174_1350 (size=31058) 2024-12-04T15:00:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-04T15:00:05,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/c35fb5dbda3a48438bfd11322e0f9dcf is 50, key is test_row_0/B:col10/1733324404706/Put/seqid=0 2024-12-04T15:00:05,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742175_1351 (size=12001) 2024-12-04T15:00:05,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324465339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:05,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:05,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324465339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:05,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324465339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:05,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:05,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/c35fb5dbda3a48438bfd11322e0f9dcf 2024-12-04T15:00:05,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/20a264b6590b46338bb837d24f1cee53 is 50, key is test_row_0/C:col10/1733324404706/Put/seqid=0 2024-12-04T15:00:05,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742176_1352 (size=12001) 2024-12-04T15:00:05,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/20a264b6590b46338bb837d24f1cee53 2024-12-04T15:00:05,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-04T15:00:05,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:05,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:05,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:05,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:05,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:05,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/44cd52a3414b465694e5949121fa40d0 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/44cd52a3414b465694e5949121fa40d0 2024-12-04T15:00:05,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324465430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:05,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324465430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:05,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:05,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/44cd52a3414b465694e5949121fa40d0, entries=200, sequenceid=97, filesize=38.6 K 2024-12-04T15:00:05,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/c35fb5dbda3a48438bfd11322e0f9dcf as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c35fb5dbda3a48438bfd11322e0f9dcf 2024-12-04T15:00:05,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c35fb5dbda3a48438bfd11322e0f9dcf, entries=150, sequenceid=97, filesize=11.7 K 2024-12-04T15:00:05,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/20a264b6590b46338bb837d24f1cee53 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/20a264b6590b46338bb837d24f1cee53 2024-12-04T15:00:05,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/20a264b6590b46338bb837d24f1cee53, entries=150, sequenceid=97, filesize=11.7 K 2024-12-04T15:00:05,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~126.28 KB/129315, heapSize ~362.27 KB/370968, currentSize=53.67 KB/54960 for e03f7dfff5f07d53203dd271e7128a0a in 764ms, sequenceid=97, compaction requested=false 2024-12-04T15:00:05,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:05,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-04T15:00:05,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:05,585 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T15:00:05,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:05,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:05,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:05,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:05,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:05,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:05,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204aaa1e5925fbc4059ba216032b5b904f7_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324404754/Put/seqid=0 2024-12-04T15:00:05,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742177_1353 (size=12154) 2024-12-04T15:00:05,686 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/e0af19c292c04778995adabf3ec6499b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/e0af19c292c04778995adabf3ec6499b 2024-12-04T15:00:05,694 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/A of e03f7dfff5f07d53203dd271e7128a0a into e0af19c292c04778995adabf3ec6499b(size=30.3 K), total size for store is 69.0 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-12-04T15:00:05,694 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:05,694 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/A, priority=13, startTime=1733324404692; duration=1sec 2024-12-04T15:00:05,694 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:05,696 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:A 2024-12-04T15:00:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-04T15:00:05,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:05,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:05,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324405873, max timestamp=1733324405896Num uniques 859; 2024-12-04T15:00:05,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T15:00:05,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324405873, max timestamp=1733324405896Num uniques 859; 2024-12-04T15:00:05,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T15:00:05,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324405873, max timestamp=1733324405896Num uniques 859; 2024-12-04T15:00:05,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T15:00:05,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324465946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:05,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324465946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:05,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:05,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324465950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:05,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324465950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:05,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324465960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:06,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:06,033 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204aaa1e5925fbc4059ba216032b5b904f7_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204aaa1e5925fbc4059ba216032b5b904f7_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:06,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/f380f889c06f4852b3bbcb6d513dc7e0, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:06,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/f380f889c06f4852b3bbcb6d513dc7e0 is 175, key is test_row_0/A:col10/1733324404754/Put/seqid=0 2024-12-04T15:00:06,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742178_1354 (size=30955) 2024-12-04T15:00:06,045 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/f380f889c06f4852b3bbcb6d513dc7e0 2024-12-04T15:00:06,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/ec6e9f5514b64004afa877852f9145a6 is 50, key is test_row_0/B:col10/1733324404754/Put/seqid=0 2024-12-04T15:00:06,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742179_1355 (size=12001) 2024-12-04T15:00:06,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324466071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:06,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324466071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:06,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324466080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:06,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:06,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:06,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:06,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:06,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324466290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324466290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:06,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:06,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:06,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:06,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:06,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:06,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:06,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324466300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:06,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:06,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:06,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:06,462 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/ec6e9f5514b64004afa877852f9145a6 2024-12-04T15:00:06,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/b9a6f9490e034fc7a531fac8aa6f8166 is 50, key is test_row_0/C:col10/1733324404754/Put/seqid=0 2024-12-04T15:00:06,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742180_1356 (size=12001) 2024-12-04T15:00:06,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324466601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:06,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324466610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:06,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324466621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:06,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-04T15:00:06,873 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/b9a6f9490e034fc7a531fac8aa6f8166 2024-12-04T15:00:06,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/f380f889c06f4852b3bbcb6d513dc7e0 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/f380f889c06f4852b3bbcb6d513dc7e0 2024-12-04T15:00:06,890 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/f380f889c06f4852b3bbcb6d513dc7e0, entries=150, sequenceid=110, filesize=30.2 K 2024-12-04T15:00:06,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/ec6e9f5514b64004afa877852f9145a6 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/ec6e9f5514b64004afa877852f9145a6 2024-12-04T15:00:06,900 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/ec6e9f5514b64004afa877852f9145a6, entries=150, sequenceid=110, filesize=11.7 K 2024-12-04T15:00:06,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/b9a6f9490e034fc7a531fac8aa6f8166 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/b9a6f9490e034fc7a531fac8aa6f8166 2024-12-04T15:00:06,916 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/b9a6f9490e034fc7a531fac8aa6f8166, entries=150, sequenceid=110, filesize=11.7 K 2024-12-04T15:00:06,917 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e03f7dfff5f07d53203dd271e7128a0a in 1332ms, sequenceid=110, compaction requested=true 2024-12-04T15:00:06,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:06,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:06,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-04T15:00:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-12-04T15:00:06,920 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-04T15:00:06,921 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2630 sec 2024-12-04T15:00:06,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.2710 sec 2024-12-04T15:00:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:06,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-04T15:00:06,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:06,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:06,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:06,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:06,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:06,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:06,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412043030f3b1fcac48d9bd9a3e0c989fe496_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324406975/Put/seqid=0 2024-12-04T15:00:07,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742181_1357 (size=14794) 2024-12-04T15:00:07,001 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:07,005 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412043030f3b1fcac48d9bd9a3e0c989fe496_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412043030f3b1fcac48d9bd9a3e0c989fe496_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:07,006 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/5e18309a541b4be788a54176f82366e7, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:07,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/5e18309a541b4be788a54176f82366e7 is 175, key is test_row_0/A:col10/1733324406975/Put/seqid=0 2024-12-04T15:00:07,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742182_1358 (size=39749) 2024-12-04T15:00:07,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324467024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:07,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:07,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324467025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:07,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324467110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:07,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:07,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324467130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:07,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:07,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324467140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:07,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:07,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324467141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:07,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:07,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:07,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324467150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:07,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:07,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324467360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:07,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:07,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324467370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:07,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:07,415 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/5e18309a541b4be788a54176f82366e7 2024-12-04T15:00:07,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/2399476653d341eaa7211f612b20b284 is 50, key is test_row_0/B:col10/1733324406975/Put/seqid=0 2024-12-04T15:00:07,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742183_1359 (size=12151) 2024-12-04T15:00:07,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324467670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:07,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324467680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:07,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:07,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:07,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/2399476653d341eaa7211f612b20b284 2024-12-04T15:00:07,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/e1abf86d7ca84a2aa55feeacc2b2c907 is 50, key is test_row_0/C:col10/1733324406975/Put/seqid=0 2024-12-04T15:00:07,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742184_1360 (size=12151) 2024-12-04T15:00:07,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/e1abf86d7ca84a2aa55feeacc2b2c907 2024-12-04T15:00:07,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/5e18309a541b4be788a54176f82366e7 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/5e18309a541b4be788a54176f82366e7 2024-12-04T15:00:07,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/5e18309a541b4be788a54176f82366e7, entries=200, sequenceid=147, filesize=38.8 K 2024-12-04T15:00:07,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/2399476653d341eaa7211f612b20b284 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/2399476653d341eaa7211f612b20b284 2024-12-04T15:00:07,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/2399476653d341eaa7211f612b20b284, entries=150, sequenceid=147, filesize=11.9 K 2024-12-04T15:00:07,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/e1abf86d7ca84a2aa55feeacc2b2c907 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/e1abf86d7ca84a2aa55feeacc2b2c907 2024-12-04T15:00:07,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/e1abf86d7ca84a2aa55feeacc2b2c907, entries=150, sequenceid=147, filesize=11.9 K 2024-12-04T15:00:07,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for e03f7dfff5f07d53203dd271e7128a0a in 878ms, sequenceid=147, compaction requested=true 2024-12-04T15:00:07,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:07,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:00:07,857 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T15:00:07,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:07,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:00:07,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:07,857 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T15:00:07,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:00:07,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:07,858 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141311 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T15:00:07,858 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T15:00:07,858 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/B is initiating minor compaction (all files) 2024-12-04T15:00:07,858 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/A is initiating minor compaction (all files) 2024-12-04T15:00:07,858 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/B in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:07,858 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/A in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:07,858 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/8b2749fc7dba4ef7a1353c99f605f074, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c35fb5dbda3a48438bfd11322e0f9dcf, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/ec6e9f5514b64004afa877852f9145a6, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/2399476653d341eaa7211f612b20b284] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=47.1 K 2024-12-04T15:00:07,858 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/e0af19c292c04778995adabf3ec6499b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/44cd52a3414b465694e5949121fa40d0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/f380f889c06f4852b3bbcb6d513dc7e0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/5e18309a541b4be788a54176f82366e7] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=138.0 K 2024-12-04T15:00:07,858 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:07,858 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/e0af19c292c04778995adabf3ec6499b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/44cd52a3414b465694e5949121fa40d0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/f380f889c06f4852b3bbcb6d513dc7e0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/5e18309a541b4be788a54176f82366e7] 2024-12-04T15:00:07,858 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b2749fc7dba4ef7a1353c99f605f074, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733324404183 2024-12-04T15:00:07,859 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting c35fb5dbda3a48438bfd11322e0f9dcf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733324404653 2024-12-04T15:00:07,859 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0af19c292c04778995adabf3ec6499b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733324404183 2024-12-04T15:00:07,859 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting ec6e9f5514b64004afa877852f9145a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733324404734 2024-12-04T15:00:07,859 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44cd52a3414b465694e5949121fa40d0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733324404653 2024-12-04T15:00:07,859 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 2399476653d341eaa7211f612b20b284, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733324405945 2024-12-04T15:00:07,859 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f380f889c06f4852b3bbcb6d513dc7e0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733324404734 2024-12-04T15:00:07,859 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e18309a541b4be788a54176f82366e7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733324405935 2024-12-04T15:00:07,866 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#B#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:07,867 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/1d3506c563734ddd81f337878ee85f6e is 50, key is test_row_0/B:col10/1733324406975/Put/seqid=0 2024-12-04T15:00:07,868 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:07,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742185_1361 (size=12391) 2024-12-04T15:00:07,875 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204c5a85742ceab4bc88808b16e140f1526_e03f7dfff5f07d53203dd271e7128a0a store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:07,876 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204c5a85742ceab4bc88808b16e140f1526_e03f7dfff5f07d53203dd271e7128a0a, store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:07,876 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c5a85742ceab4bc88808b16e140f1526_e03f7dfff5f07d53203dd271e7128a0a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:07,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742186_1362 (size=4469) 2024-12-04T15:00:08,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:08,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T15:00:08,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:08,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:08,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:08,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:08,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:08,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:08,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045e0f1b9fdd264dee97cdf895ea2510aa_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324407023/Put/seqid=0 2024-12-04T15:00:08,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742187_1363 (size=14794) 2024-12-04T15:00:08,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:08,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324468231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:08,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733324408177, max timestamp=1733324408241Num uniques 859; 2024-12-04T15:00:08,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733324408177, max timestamp=1733324408241Num uniques 859; 2024-12-04T15:00:08,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-04T15:00:08,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-04T15:00:08,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733324408177, max timestamp=1733324408241Num uniques 859; 2024-12-04T15:00:08,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-04T15:00:08,275 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/1d3506c563734ddd81f337878ee85f6e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/1d3506c563734ddd81f337878ee85f6e 2024-12-04T15:00:08,282 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/B of e03f7dfff5f07d53203dd271e7128a0a into 1d3506c563734ddd81f337878ee85f6e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:08,282 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:08,282 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/B, priority=12, startTime=1733324407857; duration=0sec 2024-12-04T15:00:08,282 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:08,282 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:B 2024-12-04T15:00:08,282 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T15:00:08,284 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T15:00:08,284 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/C is initiating minor compaction (all files) 2024-12-04T15:00:08,284 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/C in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:08,284 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/644a962ccf5a4718a5dfbf5e454e7936, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/20a264b6590b46338bb837d24f1cee53, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/b9a6f9490e034fc7a531fac8aa6f8166, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/e1abf86d7ca84a2aa55feeacc2b2c907] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=47.1 K 2024-12-04T15:00:08,284 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#A#compaction#307 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:08,285 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 644a962ccf5a4718a5dfbf5e454e7936, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733324404183 2024-12-04T15:00:08,285 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/66b9bc1632334fbf890b2b56e79e30ca is 175, key is test_row_0/A:col10/1733324406975/Put/seqid=0 2024-12-04T15:00:08,285 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 20a264b6590b46338bb837d24f1cee53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733324404653 2024-12-04T15:00:08,285 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting b9a6f9490e034fc7a531fac8aa6f8166, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733324404734 2024-12-04T15:00:08,286 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting e1abf86d7ca84a2aa55feeacc2b2c907, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733324405945 2024-12-04T15:00:08,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742188_1364 (size=31345) 2024-12-04T15:00:08,291 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/66b9bc1632334fbf890b2b56e79e30ca as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/66b9bc1632334fbf890b2b56e79e30ca 2024-12-04T15:00:08,293 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#C#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:08,294 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/3df8153fd5ea40e4a8922512058fbecd is 50, key is test_row_0/C:col10/1733324406975/Put/seqid=0 2024-12-04T15:00:08,298 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/A of e03f7dfff5f07d53203dd271e7128a0a into 66b9bc1632334fbf890b2b56e79e30ca(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:08,298 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:08,298 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/A, priority=12, startTime=1733324407857; duration=0sec 2024-12-04T15:00:08,298 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:08,298 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:A 2024-12-04T15:00:08,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742189_1365 (size=12391) 2024-12-04T15:00:08,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324468302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:08,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324468305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:08,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324468306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:08,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324468310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:08,321 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/3df8153fd5ea40e4a8922512058fbecd as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/3df8153fd5ea40e4a8922512058fbecd 2024-12-04T15:00:08,326 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/C of e03f7dfff5f07d53203dd271e7128a0a into 3df8153fd5ea40e4a8922512058fbecd(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:08,326 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:08,326 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/C, priority=12, startTime=1733324407857; duration=0sec 2024-12-04T15:00:08,326 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:08,326 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:C 2024-12-04T15:00:08,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324468350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:08,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324468421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:08,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:08,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324468430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:08,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:08,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324468430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:08,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:08,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:08,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:08,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324468431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:08,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:08,544 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:08,548 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045e0f1b9fdd264dee97cdf895ea2510aa_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045e0f1b9fdd264dee97cdf895ea2510aa_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:08,549 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/84a31991bc9b438397989aca86c30b1c, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:08,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/84a31991bc9b438397989aca86c30b1c is 175, key is test_row_0/A:col10/1733324407023/Put/seqid=0 2024-12-04T15:00:08,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742190_1366 (size=39749) 2024-12-04T15:00:08,557 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/84a31991bc9b438397989aca86c30b1c 2024-12-04T15:00:08,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/2dd651dccc934cc8964fe1d7c8a11b26 is 50, key is test_row_0/B:col10/1733324407023/Put/seqid=0 2024-12-04T15:00:08,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742191_1367 (size=12151) 2024-12-04T15:00:08,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324468570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:08,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:08,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:08,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324468630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:08,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:08,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:08,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324468641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:08,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324468641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:08,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324468650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:08,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-04T15:00:08,811 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:08,811 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:00:08,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-04T15:00:08,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T15:00:08,813 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:00:08,814 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:00:08,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:00:08,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324468890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:08,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T15:00:08,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324468950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:08,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324468960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:08,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324468960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:08,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T15:00:08,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:08,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:08,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:08,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:08,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:08,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/2dd651dccc934cc8964fe1d7c8a11b26 2024-12-04T15:00:08,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324468970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:08,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/211f2de935224991a2b1088ad86da3ac is 50, key is test_row_0/C:col10/1733324407023/Put/seqid=0 2024-12-04T15:00:08,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742192_1368 (size=12151) 2024-12-04T15:00:08,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/211f2de935224991a2b1088ad86da3ac 2024-12-04T15:00:08,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/84a31991bc9b438397989aca86c30b1c as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/84a31991bc9b438397989aca86c30b1c 2024-12-04T15:00:09,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/84a31991bc9b438397989aca86c30b1c, entries=200, sequenceid=158, filesize=38.8 K 2024-12-04T15:00:09,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/2dd651dccc934cc8964fe1d7c8a11b26 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/2dd651dccc934cc8964fe1d7c8a11b26 2024-12-04T15:00:09,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/2dd651dccc934cc8964fe1d7c8a11b26, entries=150, sequenceid=158, filesize=11.9 K 2024-12-04T15:00:09,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/211f2de935224991a2b1088ad86da3ac as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/211f2de935224991a2b1088ad86da3ac 2024-12-04T15:00:09,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/211f2de935224991a2b1088ad86da3ac, entries=150, sequenceid=158, filesize=11.9 K 2024-12-04T15:00:09,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for e03f7dfff5f07d53203dd271e7128a0a in 882ms, sequenceid=158, compaction requested=false 2024-12-04T15:00:09,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:09,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T15:00:09,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:09,119 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-12-04T15:00:09,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:09,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:09,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:09,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:09,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:09,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:09,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047d3c1259afbb44a897a37e22b7a7f057_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324408304/Put/seqid=0 2024-12-04T15:00:09,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742193_1369 (size=12304) 2024-12-04T15:00:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T15:00:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:09,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T15:00:09,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324469479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:09,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:09,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:09,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324469486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324469487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:09,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:09,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324469487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:09,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:09,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:09,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324469488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:09,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:09,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:09,534 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047d3c1259afbb44a897a37e22b7a7f057_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047d3c1259afbb44a897a37e22b7a7f057_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:09,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/dd0b10c047dc4b95ada36f66d2242d67, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:09,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/dd0b10c047dc4b95ada36f66d2242d67 is 175, key is test_row_0/A:col10/1733324408304/Put/seqid=0 2024-12-04T15:00:09,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742194_1370 (size=31105) 2024-12-04T15:00:09,544 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/dd0b10c047dc4b95ada36f66d2242d67 2024-12-04T15:00:09,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/a3f2d38d57de40a78b67460f7d5200a8 is 50, key is test_row_0/B:col10/1733324408304/Put/seqid=0 2024-12-04T15:00:09,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742195_1371 (size=12151) 2024-12-04T15:00:09,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324469610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:09,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:09,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324469610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:09,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:09,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324469610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:09,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:09,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:09,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324469610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:09,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:09,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:09,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324469820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:09,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324469821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:09,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:09,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324469830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:09,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:09,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324469830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:09,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T15:00:09,970 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/a3f2d38d57de40a78b67460f7d5200a8 2024-12-04T15:00:09,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/cd2db5594c084ad68e86983cfcef6809 is 50, key is test_row_0/C:col10/1733324408304/Put/seqid=0 2024-12-04T15:00:09,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742196_1372 (size=12151) 2024-12-04T15:00:09,995 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/cd2db5594c084ad68e86983cfcef6809 2024-12-04T15:00:10,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/dd0b10c047dc4b95ada36f66d2242d67 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/dd0b10c047dc4b95ada36f66d2242d67 2024-12-04T15:00:10,005 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/dd0b10c047dc4b95ada36f66d2242d67, entries=150, sequenceid=194, filesize=30.4 K 2024-12-04T15:00:10,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/a3f2d38d57de40a78b67460f7d5200a8 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/a3f2d38d57de40a78b67460f7d5200a8 2024-12-04T15:00:10,017 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/a3f2d38d57de40a78b67460f7d5200a8, entries=150, sequenceid=194, filesize=11.9 K 2024-12-04T15:00:10,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/cd2db5594c084ad68e86983cfcef6809 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/cd2db5594c084ad68e86983cfcef6809 2024-12-04T15:00:10,023 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/cd2db5594c084ad68e86983cfcef6809, entries=150, sequenceid=194, filesize=11.9 K 2024-12-04T15:00:10,024 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=60.38 KB/61830 for e03f7dfff5f07d53203dd271e7128a0a in 905ms, sequenceid=194, compaction requested=true 2024-12-04T15:00:10,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:10,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:10,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-04T15:00:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-04T15:00:10,033 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-04T15:00:10,033 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2120 sec 2024-12-04T15:00:10,035 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.2230 sec 2024-12-04T15:00:10,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:10,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T15:00:10,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:10,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:10,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:10,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:10,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:10,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:10,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204536d63ee54e54ce2afb5a66c90a9a02c_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324409486/Put/seqid=0 2024-12-04T15:00:10,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742197_1373 (size=14794) 2024-12-04T15:00:10,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324470171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:10,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324470176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:10,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324470180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:10,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:10,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:10,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324470181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:10,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:10,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324470290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:10,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324470290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324470290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:10,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324470300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:10,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324470500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:10,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324470500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:10,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324470500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:10,505 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in e03f7dfff5f07d53203dd271e7128a0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:10,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324470500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:10,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:10,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:10,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:10,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324470510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:10,556 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:10,568 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204536d63ee54e54ce2afb5a66c90a9a02c_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204536d63ee54e54ce2afb5a66c90a9a02c_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:10,569 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/ac231970ee5e472a8dc79ae6680675a0, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:10,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/ac231970ee5e472a8dc79ae6680675a0 is 175, key is test_row_0/A:col10/1733324409486/Put/seqid=0 2024-12-04T15:00:10,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742198_1374 (size=39749) 2024-12-04T15:00:10,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324470811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324470811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:10,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:10,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:10,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324470811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:10,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:10,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:10,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324470820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:10,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:10,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T15:00:10,961 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:10,961 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:00:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-04T15:00:10,963 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:00:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T15:00:10,964 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:00:10,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:00:11,017 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/ac231970ee5e472a8dc79ae6680675a0 2024-12-04T15:00:11,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/9f55cb45fa3a45c892ed89750b852f1b is 50, key is test_row_0/B:col10/1733324409486/Put/seqid=0 2024-12-04T15:00:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T15:00:11,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742199_1375 (size=12151) 2024-12-04T15:00:11,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T15:00:11,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:11,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:11,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:11,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:11,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:11,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T15:00:11,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:11,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:11,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:11,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:11,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T15:00:11,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324471320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:11,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:11,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:11,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324471320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:11,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:11,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:11,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:11,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:11,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324471330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:11,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:11,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:11,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:11,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324471340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:11,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:11,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:11,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:11,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T15:00:11,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:11,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:11,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:11,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:11,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:11,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/9f55cb45fa3a45c892ed89750b852f1b 2024-12-04T15:00:11,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/9c60eace0cd548fa9432c99e6bc37de2 is 50, key is test_row_0/C:col10/1733324409486/Put/seqid=0 2024-12-04T15:00:11,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742200_1376 (size=12151) 2024-12-04T15:00:11,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T15:00:11,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:11,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:11,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:11,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:11,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:11,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T15:00:11,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T15:00:11,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:11,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:11,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:11,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:11,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:11,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:11,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T15:00:11,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:11,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:11,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:11,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:11,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:11,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/9c60eace0cd548fa9432c99e6bc37de2 2024-12-04T15:00:11,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/ac231970ee5e472a8dc79ae6680675a0 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/ac231970ee5e472a8dc79ae6680675a0 2024-12-04T15:00:11,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/ac231970ee5e472a8dc79ae6680675a0, entries=200, sequenceid=207, filesize=38.8 K 2024-12-04T15:00:11,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/9f55cb45fa3a45c892ed89750b852f1b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/9f55cb45fa3a45c892ed89750b852f1b 2024-12-04T15:00:11,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/9f55cb45fa3a45c892ed89750b852f1b, entries=150, sequenceid=207, filesize=11.9 K 2024-12-04T15:00:11,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/9c60eace0cd548fa9432c99e6bc37de2 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/9c60eace0cd548fa9432c99e6bc37de2 2024-12-04T15:00:11,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/9c60eace0cd548fa9432c99e6bc37de2, entries=150, sequenceid=207, filesize=11.9 K 2024-12-04T15:00:11,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e03f7dfff5f07d53203dd271e7128a0a in 1804ms, sequenceid=207, compaction requested=true 2024-12-04T15:00:11,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:11,936 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T15:00:11,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:00:11,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:11,937 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T15:00:11,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:00:11,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:11,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:00:11,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:11,938 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141948 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T15:00:11,938 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/A is initiating minor compaction (all files) 2024-12-04T15:00:11,938 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/A in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:11,938 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/66b9bc1632334fbf890b2b56e79e30ca, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/84a31991bc9b438397989aca86c30b1c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/dd0b10c047dc4b95ada36f66d2242d67, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/ac231970ee5e472a8dc79ae6680675a0] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=138.6 K 2024-12-04T15:00:11,938 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:11,938 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/66b9bc1632334fbf890b2b56e79e30ca, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/84a31991bc9b438397989aca86c30b1c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/dd0b10c047dc4b95ada36f66d2242d67, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/ac231970ee5e472a8dc79ae6680675a0] 2024-12-04T15:00:11,939 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66b9bc1632334fbf890b2b56e79e30ca, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733324405945 2024-12-04T15:00:11,939 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T15:00:11,940 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/B is initiating minor compaction (all files) 2024-12-04T15:00:11,940 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/B in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:11,940 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/1d3506c563734ddd81f337878ee85f6e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/2dd651dccc934cc8964fe1d7c8a11b26, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/a3f2d38d57de40a78b67460f7d5200a8, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/9f55cb45fa3a45c892ed89750b852f1b] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=47.7 K 2024-12-04T15:00:11,941 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d3506c563734ddd81f337878ee85f6e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733324405945 2024-12-04T15:00:11,941 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84a31991bc9b438397989aca86c30b1c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733324407010 2024-12-04T15:00:11,941 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dd651dccc934cc8964fe1d7c8a11b26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733324407010 2024-12-04T15:00:11,941 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd0b10c047dc4b95ada36f66d2242d67, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733324408262 2024-12-04T15:00:11,941 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting a3f2d38d57de40a78b67460f7d5200a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733324408262 2024-12-04T15:00:11,941 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac231970ee5e472a8dc79ae6680675a0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733324409449 2024-12-04T15:00:11,942 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f55cb45fa3a45c892ed89750b852f1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733324409449 2024-12-04T15:00:11,961 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#B#compaction#318 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:11,962 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/14faa8bbe90d48e186eadf451128bf2b is 50, key is test_row_0/B:col10/1733324409486/Put/seqid=0 2024-12-04T15:00:11,970 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:11,988 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204cc4e87390db74b1ba4e094a8ee9966c4_e03f7dfff5f07d53203dd271e7128a0a store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:11,990 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204cc4e87390db74b1ba4e094a8ee9966c4_e03f7dfff5f07d53203dd271e7128a0a, store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:11,990 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204cc4e87390db74b1ba4e094a8ee9966c4_e03f7dfff5f07d53203dd271e7128a0a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:12,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742201_1377 (size=12527) 2024-12-04T15:00:12,034 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/14faa8bbe90d48e186eadf451128bf2b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/14faa8bbe90d48e186eadf451128bf2b 2024-12-04T15:00:12,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742202_1378 (size=4469) 2024-12-04T15:00:12,038 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#A#compaction#319 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:12,039 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/07e6089445ed49d68166342b94571279 is 175, key is test_row_0/A:col10/1733324409486/Put/seqid=0 2024-12-04T15:00:12,039 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/B of e03f7dfff5f07d53203dd271e7128a0a into 14faa8bbe90d48e186eadf451128bf2b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:12,039 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:12,039 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/B, priority=12, startTime=1733324411937; duration=0sec 2024-12-04T15:00:12,039 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:12,039 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:B 2024-12-04T15:00:12,040 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T15:00:12,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T15:00:12,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:12,041 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-04T15:00:12,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:12,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:12,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:12,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:12,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:12,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:12,056 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T15:00:12,056 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/C is initiating minor compaction (all files) 2024-12-04T15:00:12,056 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/C in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:12,056 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/3df8153fd5ea40e4a8922512058fbecd, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/211f2de935224991a2b1088ad86da3ac, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/cd2db5594c084ad68e86983cfcef6809, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/9c60eace0cd548fa9432c99e6bc37de2] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=47.7 K 2024-12-04T15:00:12,057 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 3df8153fd5ea40e4a8922512058fbecd, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733324405945 2024-12-04T15:00:12,060 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 211f2de935224991a2b1088ad86da3ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733324407010 2024-12-04T15:00:12,060 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting cd2db5594c084ad68e86983cfcef6809, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733324408262 2024-12-04T15:00:12,062 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c60eace0cd548fa9432c99e6bc37de2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733324409449 2024-12-04T15:00:12,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204eb52c648df8d4aa28892b5a64465b0f4_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324410170/Put/seqid=0 2024-12-04T15:00:12,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742203_1379 (size=31481) 2024-12-04T15:00:12,083 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#C#compaction#321 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:12,083 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/efdb826eab0f46e896bd9cb98f4ce81b is 50, key is test_row_0/C:col10/1733324409486/Put/seqid=0 2024-12-04T15:00:12,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742204_1380 (size=12304) 2024-12-04T15:00:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T15:00:12,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742205_1381 (size=12527) 2024-12-04T15:00:12,140 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/efdb826eab0f46e896bd9cb98f4ce81b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/efdb826eab0f46e896bd9cb98f4ce81b 2024-12-04T15:00:12,145 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/C of e03f7dfff5f07d53203dd271e7128a0a into efdb826eab0f46e896bd9cb98f4ce81b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:12,145 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:12,145 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/C, priority=12, startTime=1733324411937; duration=0sec 2024-12-04T15:00:12,146 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:12,146 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:C 2024-12-04T15:00:12,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:12,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324472370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:12,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324472371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324472373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:12,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324472374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:12,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:12,483 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/07e6089445ed49d68166342b94571279 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/07e6089445ed49d68166342b94571279 2024-12-04T15:00:12,491 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/A of e03f7dfff5f07d53203dd271e7128a0a into 07e6089445ed49d68166342b94571279(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:12,491 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:12,491 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/A, priority=12, startTime=1733324411936; duration=0sec 2024-12-04T15:00:12,491 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:12,491 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:A 2024-12-04T15:00:12,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:12,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324472490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:12,499 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204eb52c648df8d4aa28892b5a64465b0f4_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204eb52c648df8d4aa28892b5a64465b0f4_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:12,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:12,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324472490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:12,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324472490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:12,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324472490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:12,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/26ce439201e94de3afcf30af94c49815, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:12,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/26ce439201e94de3afcf30af94c49815 is 175, key is test_row_0/A:col10/1733324410170/Put/seqid=0 2024-12-04T15:00:12,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742206_1382 (size=31105) 2024-12-04T15:00:12,504 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/26ce439201e94de3afcf30af94c49815 2024-12-04T15:00:12,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/148e5a59aa0642c489ea33cad4750975 is 50, key is test_row_0/B:col10/1733324410170/Put/seqid=0 2024-12-04T15:00:12,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742207_1383 (size=12151) 2024-12-04T15:00:12,513 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/148e5a59aa0642c489ea33cad4750975 2024-12-04T15:00:12,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/f47b4c76b44a480782f14a943ac9954e is 50, key is test_row_0/C:col10/1733324410170/Put/seqid=0 2024-12-04T15:00:12,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742208_1384 (size=12151) 2024-12-04T15:00:12,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324472520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:12,525 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in e03f7dfff5f07d53203dd271e7128a0a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:12,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324472710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:12,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:12,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324472710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:12,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324472710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:12,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324472710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:12,921 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/f47b4c76b44a480782f14a943ac9954e 2024-12-04T15:00:12,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/26ce439201e94de3afcf30af94c49815 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/26ce439201e94de3afcf30af94c49815 2024-12-04T15:00:12,928 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/26ce439201e94de3afcf30af94c49815, entries=150, sequenceid=227, filesize=30.4 K 2024-12-04T15:00:12,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/148e5a59aa0642c489ea33cad4750975 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/148e5a59aa0642c489ea33cad4750975 2024-12-04T15:00:12,931 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/148e5a59aa0642c489ea33cad4750975, entries=150, sequenceid=227, filesize=11.9 K 2024-12-04T15:00:12,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/f47b4c76b44a480782f14a943ac9954e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/f47b4c76b44a480782f14a943ac9954e 2024-12-04T15:00:12,934 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/f47b4c76b44a480782f14a943ac9954e, entries=150, sequenceid=227, filesize=11.9 K 2024-12-04T15:00:12,935 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e03f7dfff5f07d53203dd271e7128a0a in 894ms, sequenceid=227, compaction requested=false 2024-12-04T15:00:12,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:12,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:12,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-04T15:00:12,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-04T15:00:12,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-04T15:00:12,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9720 sec 2024-12-04T15:00:12,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.9770 sec 2024-12-04T15:00:13,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:13,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-04T15:00:13,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:13,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:13,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:13,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:13,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:13,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:13,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e1c27340e10e4fa9b85a0765806db51f_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324413023/Put/seqid=0 2024-12-04T15:00:13,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742209_1385 (size=14794) 2024-12-04T15:00:13,050 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:13,054 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e1c27340e10e4fa9b85a0765806db51f_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e1c27340e10e4fa9b85a0765806db51f_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:13,055 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/997868d0a90847109c5eaa2672ab99a6, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:13,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/997868d0a90847109c5eaa2672ab99a6 is 175, key is test_row_0/A:col10/1733324413023/Put/seqid=0 2024-12-04T15:00:13,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:13,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324473048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:13,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742210_1386 (size=39749) 2024-12-04T15:00:13,074 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/997868d0a90847109c5eaa2672ab99a6 2024-12-04T15:00:13,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:13,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324473064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:13,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:13,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324473064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:13,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:13,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324473064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:13,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/8be9cb19ef804fd2a39ce2434421a8f7 is 50, key is test_row_0/B:col10/1733324413023/Put/seqid=0 2024-12-04T15:00:13,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742211_1387 (size=12151) 2024-12-04T15:00:13,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/8be9cb19ef804fd2a39ce2434421a8f7 2024-12-04T15:00:13,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T15:00:13,110 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:13,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:00:13,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-04T15:00:13,113 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:00:13,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T15:00:13,114 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:00:13,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:00:13,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/fa3f1a139c224072b013508a477b75a4 is 50, key is test_row_0/C:col10/1733324413023/Put/seqid=0 2024-12-04T15:00:13,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742212_1388 (size=12151) 2024-12-04T15:00:13,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/fa3f1a139c224072b013508a477b75a4 2024-12-04T15:00:13,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/997868d0a90847109c5eaa2672ab99a6 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/997868d0a90847109c5eaa2672ab99a6 2024-12-04T15:00:13,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/997868d0a90847109c5eaa2672ab99a6, entries=200, sequenceid=245, filesize=38.8 K 2024-12-04T15:00:13,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/8be9cb19ef804fd2a39ce2434421a8f7 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/8be9cb19ef804fd2a39ce2434421a8f7 2024-12-04T15:00:13,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/8be9cb19ef804fd2a39ce2434421a8f7, entries=150, sequenceid=245, filesize=11.9 K 2024-12-04T15:00:13,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/fa3f1a139c224072b013508a477b75a4 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/fa3f1a139c224072b013508a477b75a4 2024-12-04T15:00:13,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/fa3f1a139c224072b013508a477b75a4, entries=150, sequenceid=245, filesize=11.9 K 2024-12-04T15:00:13,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for e03f7dfff5f07d53203dd271e7128a0a in 134ms, sequenceid=245, compaction requested=true 2024-12-04T15:00:13,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:13,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:00:13,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:13,158 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:13,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:00:13,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:13,158 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:13,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:00:13,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:13,159 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:13,159 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:13,159 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/A is initiating minor compaction (all files) 2024-12-04T15:00:13,159 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/B is initiating minor compaction (all files) 2024-12-04T15:00:13,159 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/A in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:13,159 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/B in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:13,159 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/14faa8bbe90d48e186eadf451128bf2b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/148e5a59aa0642c489ea33cad4750975, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/8be9cb19ef804fd2a39ce2434421a8f7] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=36.0 K 2024-12-04T15:00:13,159 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/07e6089445ed49d68166342b94571279, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/26ce439201e94de3afcf30af94c49815, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/997868d0a90847109c5eaa2672ab99a6] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=99.9 K 2024-12-04T15:00:13,159 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:13,159 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/07e6089445ed49d68166342b94571279, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/26ce439201e94de3afcf30af94c49815, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/997868d0a90847109c5eaa2672ab99a6] 2024-12-04T15:00:13,160 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07e6089445ed49d68166342b94571279, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733324409449 2024-12-04T15:00:13,160 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26ce439201e94de3afcf30af94c49815, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733324410168 2024-12-04T15:00:13,160 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 14faa8bbe90d48e186eadf451128bf2b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733324409449 2024-12-04T15:00:13,160 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 997868d0a90847109c5eaa2672ab99a6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733324412367 2024-12-04T15:00:13,160 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 148e5a59aa0642c489ea33cad4750975, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733324410168 2024-12-04T15:00:13,162 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 8be9cb19ef804fd2a39ce2434421a8f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733324412367 2024-12-04T15:00:13,168 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#B#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:13,169 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/273be2c0db2f478590487f934a9d149f is 50, key is test_row_0/B:col10/1733324413023/Put/seqid=0 2024-12-04T15:00:13,175 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:13,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:13,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-04T15:00:13,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:13,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:13,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:13,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:13,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:13,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:13,183 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412041302428dd1914a1fbcd341abd1ecd22b_e03f7dfff5f07d53203dd271e7128a0a store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:13,185 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412041302428dd1914a1fbcd341abd1ecd22b_e03f7dfff5f07d53203dd271e7128a0a, store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:13,185 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041302428dd1914a1fbcd341abd1ecd22b_e03f7dfff5f07d53203dd271e7128a0a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:13,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041304fe8fb3cf4663840033cc2dc5a3be_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324413176/Put/seqid=0 2024-12-04T15:00:13,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742213_1389 (size=12629) 2024-12-04T15:00:13,208 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/273be2c0db2f478590487f934a9d149f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/273be2c0db2f478590487f934a9d149f 2024-12-04T15:00:13,213 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/B of e03f7dfff5f07d53203dd271e7128a0a into 273be2c0db2f478590487f934a9d149f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:13,213 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:13,214 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/B, priority=13, startTime=1733324413158; duration=0sec 2024-12-04T15:00:13,214 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:13,214 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:B 2024-12-04T15:00:13,214 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:13,214 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:13,215 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/C is initiating minor compaction (all files) 2024-12-04T15:00:13,215 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/C in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:13,215 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/efdb826eab0f46e896bd9cb98f4ce81b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/f47b4c76b44a480782f14a943ac9954e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/fa3f1a139c224072b013508a477b75a4] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=36.0 K 2024-12-04T15:00:13,215 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting efdb826eab0f46e896bd9cb98f4ce81b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733324409449 2024-12-04T15:00:13,215 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting f47b4c76b44a480782f14a943ac9954e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733324410168 2024-12-04T15:00:13,216 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting fa3f1a139c224072b013508a477b75a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733324412367 2024-12-04T15:00:13,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:13,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324473207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:13,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742214_1390 (size=4469) 2024-12-04T15:00:13,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324473208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:13,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324473215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:13,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324473217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:13,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:13,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T15:00:13,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742215_1391 (size=14994) 2024-12-04T15:00:13,224 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:13,226 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#C#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:13,227 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/d23049b824134779a32abbc587eeb1f6 is 50, key is test_row_0/C:col10/1733324413023/Put/seqid=0 2024-12-04T15:00:13,234 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041304fe8fb3cf4663840033cc2dc5a3be_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041304fe8fb3cf4663840033cc2dc5a3be_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:13,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742216_1392 (size=12629) 2024-12-04T15:00:13,237 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/da702713cc3e45c7805b0f6b143a66d1, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:13,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/da702713cc3e45c7805b0f6b143a66d1 is 175, key is test_row_0/A:col10/1733324413176/Put/seqid=0 2024-12-04T15:00:13,243 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/d23049b824134779a32abbc587eeb1f6 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/d23049b824134779a32abbc587eeb1f6 2024-12-04T15:00:13,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742217_1393 (size=39949) 2024-12-04T15:00:13,249 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/da702713cc3e45c7805b0f6b143a66d1 2024-12-04T15:00:13,249 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/C of e03f7dfff5f07d53203dd271e7128a0a into d23049b824134779a32abbc587eeb1f6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:13,249 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:13,249 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/C, priority=13, startTime=1733324413158; duration=0sec 2024-12-04T15:00:13,250 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:13,250 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:C 2024-12-04T15:00:13,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/4250f3f1782a4156a421321bd4e88eb3 is 50, key is test_row_0/B:col10/1733324413176/Put/seqid=0 2024-12-04T15:00:13,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742218_1394 (size=12301) 2024-12-04T15:00:13,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-04T15:00:13,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:13,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:13,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:13,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:13,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:13,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:13,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324473330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324473330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324473330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324473330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:13,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-04T15:00:13,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:13,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:13,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:13,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:13,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:13,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T15:00:13,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324473540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:13,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:13,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324473540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:13,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:13,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324473540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:13,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:13,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:13,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:13,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324473541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:13,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:13,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-04T15:00:13,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:13,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:13,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:13,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:13,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:13,619 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#A#compaction#328 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:13,620 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/06b08331f1a047fdae121860012cee4a is 175, key is test_row_0/A:col10/1733324413023/Put/seqid=0 2024-12-04T15:00:13,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742219_1395 (size=31583) 2024-12-04T15:00:13,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/4250f3f1782a4156a421321bd4e88eb3 2024-12-04T15:00:13,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/0d0b219f56644a2ea6a16ab4083f0bb9 is 50, key is test_row_0/C:col10/1733324413176/Put/seqid=0 2024-12-04T15:00:13,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742220_1396 (size=12301) 2024-12-04T15:00:13,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/0d0b219f56644a2ea6a16ab4083f0bb9 2024-12-04T15:00:13,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/da702713cc3e45c7805b0f6b143a66d1 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/da702713cc3e45c7805b0f6b143a66d1 2024-12-04T15:00:13,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/da702713cc3e45c7805b0f6b143a66d1, entries=200, sequenceid=263, filesize=39.0 K 2024-12-04T15:00:13,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/4250f3f1782a4156a421321bd4e88eb3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/4250f3f1782a4156a421321bd4e88eb3 2024-12-04T15:00:13,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/4250f3f1782a4156a421321bd4e88eb3, entries=150, sequenceid=263, filesize=12.0 K 2024-12-04T15:00:13,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/0d0b219f56644a2ea6a16ab4083f0bb9 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/0d0b219f56644a2ea6a16ab4083f0bb9 2024-12-04T15:00:13,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/0d0b219f56644a2ea6a16ab4083f0bb9, entries=150, sequenceid=263, filesize=12.0 K 2024-12-04T15:00:13,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e03f7dfff5f07d53203dd271e7128a0a in 504ms, sequenceid=263, compaction requested=false 2024-12-04T15:00:13,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:13,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-04T15:00:13,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:13,725 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-04T15:00:13,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:13,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:13,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:13,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:13,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:13,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:13,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204827386d5c506429888b935d140a185e7_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324413206/Put/seqid=0 2024-12-04T15:00:13,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742221_1397 (size=12454) 2024-12-04T15:00:13,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T15:00:13,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:13,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:13,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:13,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324473872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:13,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:13,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:13,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:13,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324473880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:13,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324473881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:13,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324473882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:13,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324473990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:14,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324474000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:14,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324474000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324474000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:14,026 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/06b08331f1a047fdae121860012cee4a as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/06b08331f1a047fdae121860012cee4a 2024-12-04T15:00:14,030 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/A of e03f7dfff5f07d53203dd271e7128a0a into 06b08331f1a047fdae121860012cee4a(size=30.8 K), total size for store is 69.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:14,030 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:14,030 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/A, priority=13, startTime=1733324413158; duration=0sec 2024-12-04T15:00:14,030 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:14,030 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:A 2024-12-04T15:00:14,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:14,136 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204827386d5c506429888b935d140a185e7_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204827386d5c506429888b935d140a185e7_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:14,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/e6220b8ffa224ea99e237f7ef5bd5b68, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:14,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/e6220b8ffa224ea99e237f7ef5bd5b68 is 175, key is test_row_0/A:col10/1733324413206/Put/seqid=0 2024-12-04T15:00:14,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742222_1398 (size=31255) 2024-12-04T15:00:14,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324474200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:14,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:14,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324474209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:14,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:14,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324474210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:14,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324474210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T15:00:14,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324474510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:14,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324474520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:14,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:14,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324474520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:14,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:14,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:14,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324474520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:14,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:14,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:14,541 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/e6220b8ffa224ea99e237f7ef5bd5b68 2024-12-04T15:00:14,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/7f2a660307fa40759b0433e847fb89ad is 50, key is test_row_0/B:col10/1733324413206/Put/seqid=0 2024-12-04T15:00:14,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742223_1399 (size=12301) 2024-12-04T15:00:14,949 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/7f2a660307fa40759b0433e847fb89ad 2024-12-04T15:00:14,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/f35dfcc461cb46a580aad9c31278cae4 is 50, key is test_row_0/C:col10/1733324413206/Put/seqid=0 2024-12-04T15:00:14,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742224_1400 (size=12301) 2024-12-04T15:00:14,958 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/f35dfcc461cb46a580aad9c31278cae4 2024-12-04T15:00:14,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/e6220b8ffa224ea99e237f7ef5bd5b68 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/e6220b8ffa224ea99e237f7ef5bd5b68 2024-12-04T15:00:14,966 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/e6220b8ffa224ea99e237f7ef5bd5b68, entries=150, sequenceid=279, filesize=30.5 K 2024-12-04T15:00:14,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/7f2a660307fa40759b0433e847fb89ad as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/7f2a660307fa40759b0433e847fb89ad 2024-12-04T15:00:14,969 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/7f2a660307fa40759b0433e847fb89ad, entries=150, sequenceid=279, filesize=12.0 K 2024-12-04T15:00:14,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/f35dfcc461cb46a580aad9c31278cae4 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/f35dfcc461cb46a580aad9c31278cae4 2024-12-04T15:00:14,972 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/f35dfcc461cb46a580aad9c31278cae4, entries=150, sequenceid=279, filesize=12.0 K 2024-12-04T15:00:14,974 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e03f7dfff5f07d53203dd271e7128a0a in 1249ms, sequenceid=279, compaction requested=true 2024-12-04T15:00:14,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:14,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:14,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-04T15:00:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-04T15:00:14,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-04T15:00:14,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8610 sec 2024-12-04T15:00:14,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8680 sec 2024-12-04T15:00:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:15,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-04T15:00:15,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:15,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:15,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:15,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:15,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:15,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:15,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c8476ae7a4064ae489308745fca2f98b_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324413881/Put/seqid=0 2024-12-04T15:00:15,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742225_1401 (size=14994) 2024-12-04T15:00:15,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324475046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:15,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:15,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324475050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:15,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:15,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:15,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324475052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:15,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324475055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:15,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324475161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:15,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324475161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:15,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:15,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:15,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324475171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:15,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:15,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324475171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:15,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T15:00:15,261 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:15,261 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:00:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-04T15:00:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T15:00:15,263 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:00:15,263 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:00:15,264 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:00:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T15:00:15,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:15,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324475380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:15,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324475380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:15,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324475380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:15,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:15,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324475380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:15,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:15,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T15:00:15,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:15,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:15,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:15,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:15,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:15,444 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:15,446 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c8476ae7a4064ae489308745fca2f98b_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c8476ae7a4064ae489308745fca2f98b_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:15,447 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/ba1020932bad4f56ab6458147fc08661, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:15,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/ba1020932bad4f56ab6458147fc08661 is 175, key is test_row_0/A:col10/1733324413881/Put/seqid=0 2024-12-04T15:00:15,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742226_1402 (size=39949) 2024-12-04T15:00:15,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T15:00:15,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:15,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:15,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:15,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:15,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T15:00:15,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324475700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:15,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324475701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:15,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:15,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324475701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:15,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:15,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324475701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:15,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T15:00:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:15,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:15,852 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/ba1020932bad4f56ab6458147fc08661 2024-12-04T15:00:15,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/b6ecf3f70d51460db498888a1b6c9ee4 is 50, key is test_row_0/B:col10/1733324413881/Put/seqid=0 2024-12-04T15:00:15,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T15:00:15,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:15,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:15,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:15,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:15,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T15:00:15,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742227_1403 (size=12301) 2024-12-04T15:00:16,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T15:00:16,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:16,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:16,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:16,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:16,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:16,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:16,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T15:00:16,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:16,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:16,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:16,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:16,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:16,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:16,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324476211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:16,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:16,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:16,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:16,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324476211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:16,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324476211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:16,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324476221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:16,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:16,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:16,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:16,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/b6ecf3f70d51460db498888a1b6c9ee4 2024-12-04T15:00:16,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/eaeb1a4606c040cea546d58f2cdc65c3 is 50, key is test_row_0/C:col10/1733324413881/Put/seqid=0 2024-12-04T15:00:16,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742228_1404 (size=12301) 2024-12-04T15:00:16,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/eaeb1a4606c040cea546d58f2cdc65c3 2024-12-04T15:00:16,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/ba1020932bad4f56ab6458147fc08661 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/ba1020932bad4f56ab6458147fc08661 2024-12-04T15:00:16,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T15:00:16,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:16,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:16,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:16,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:16,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:16,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/ba1020932bad4f56ab6458147fc08661, entries=200, sequenceid=299, filesize=39.0 K 2024-12-04T15:00:16,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/b6ecf3f70d51460db498888a1b6c9ee4 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/b6ecf3f70d51460db498888a1b6c9ee4 2024-12-04T15:00:16,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/b6ecf3f70d51460db498888a1b6c9ee4, entries=150, sequenceid=299, filesize=12.0 K 2024-12-04T15:00:16,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/eaeb1a4606c040cea546d58f2cdc65c3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/eaeb1a4606c040cea546d58f2cdc65c3 2024-12-04T15:00:16,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/eaeb1a4606c040cea546d58f2cdc65c3, entries=150, sequenceid=299, filesize=12.0 K 2024-12-04T15:00:16,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e03f7dfff5f07d53203dd271e7128a0a in 1324ms, sequenceid=299, compaction requested=true 2024-12-04T15:00:16,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:16,359 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T15:00:16,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:00:16,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:16,360 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T15:00:16,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:00:16,361 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142736 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T15:00:16,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:16,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:00:16,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:16,361 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/A is initiating minor compaction (all files) 2024-12-04T15:00:16,361 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/A in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:16,361 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/06b08331f1a047fdae121860012cee4a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/da702713cc3e45c7805b0f6b143a66d1, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/e6220b8ffa224ea99e237f7ef5bd5b68, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/ba1020932bad4f56ab6458147fc08661] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=139.4 K 2024-12-04T15:00:16,361 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:16,361 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/06b08331f1a047fdae121860012cee4a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/da702713cc3e45c7805b0f6b143a66d1, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/e6220b8ffa224ea99e237f7ef5bd5b68, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/ba1020932bad4f56ab6458147fc08661] 2024-12-04T15:00:16,362 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06b08331f1a047fdae121860012cee4a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733324412367 2024-12-04T15:00:16,362 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T15:00:16,362 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting da702713cc3e45c7805b0f6b143a66d1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733324413045 2024-12-04T15:00:16,362 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/B is initiating minor compaction (all files) 2024-12-04T15:00:16,362 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/B in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:16,362 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/273be2c0db2f478590487f934a9d149f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/4250f3f1782a4156a421321bd4e88eb3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/7f2a660307fa40759b0433e847fb89ad, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/b6ecf3f70d51460db498888a1b6c9ee4] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=48.4 K 2024-12-04T15:00:16,363 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 273be2c0db2f478590487f934a9d149f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733324412367 2024-12-04T15:00:16,363 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6220b8ffa224ea99e237f7ef5bd5b68, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733324413200 2024-12-04T15:00:16,363 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 4250f3f1782a4156a421321bd4e88eb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733324413045 2024-12-04T15:00:16,363 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba1020932bad4f56ab6458147fc08661, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733324413879 2024-12-04T15:00:16,363 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f2a660307fa40759b0433e847fb89ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733324413200 2024-12-04T15:00:16,365 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting b6ecf3f70d51460db498888a1b6c9ee4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733324413880 2024-12-04T15:00:16,373 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#B#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:16,374 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/1f33b79fb0ca40c1ae7aeb6e08661845 is 50, key is test_row_0/B:col10/1733324413881/Put/seqid=0 2024-12-04T15:00:16,379 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:16,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742229_1405 (size=12915) 2024-12-04T15:00:16,399 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204084d2d6e23a9494b83c8d395e5b4e3d4_e03f7dfff5f07d53203dd271e7128a0a store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:16,399 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/1f33b79fb0ca40c1ae7aeb6e08661845 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/1f33b79fb0ca40c1ae7aeb6e08661845 2024-12-04T15:00:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T15:00:16,401 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204084d2d6e23a9494b83c8d395e5b4e3d4_e03f7dfff5f07d53203dd271e7128a0a, store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:16,402 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204084d2d6e23a9494b83c8d395e5b4e3d4_e03f7dfff5f07d53203dd271e7128a0a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:16,407 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/B of e03f7dfff5f07d53203dd271e7128a0a into 1f33b79fb0ca40c1ae7aeb6e08661845(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:16,408 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:16,408 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/B, priority=12, startTime=1733324416360; duration=0sec 2024-12-04T15:00:16,408 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:16,408 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:B 2024-12-04T15:00:16,408 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T15:00:16,410 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T15:00:16,410 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/C is initiating minor compaction (all files) 2024-12-04T15:00:16,410 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/C in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:16,410 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/d23049b824134779a32abbc587eeb1f6, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/0d0b219f56644a2ea6a16ab4083f0bb9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/f35dfcc461cb46a580aad9c31278cae4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/eaeb1a4606c040cea546d58f2cdc65c3] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=48.4 K 2024-12-04T15:00:16,410 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting d23049b824134779a32abbc587eeb1f6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733324412367 2024-12-04T15:00:16,411 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d0b219f56644a2ea6a16ab4083f0bb9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733324413045 2024-12-04T15:00:16,411 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting f35dfcc461cb46a580aad9c31278cae4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733324413200 2024-12-04T15:00:16,411 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting eaeb1a4606c040cea546d58f2cdc65c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733324413880 2024-12-04T15:00:16,437 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#C#compaction#341 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:16,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742230_1406 (size=4469) 2024-12-04T15:00:16,438 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/bf69a802ee1147d08fa0a82d11b88264 is 50, key is test_row_0/C:col10/1733324413881/Put/seqid=0 2024-12-04T15:00:16,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742231_1407 (size=12915) 2024-12-04T15:00:16,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T15:00:16,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:16,496 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T15:00:16,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:16,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:16,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:16,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:16,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:16,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:16,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204902fdc91b5624969b8bef9d345b810f1_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324415049/Put/seqid=0 2024-12-04T15:00:16,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742232_1408 (size=12454) 2024-12-04T15:00:16,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:16,549 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204902fdc91b5624969b8bef9d345b810f1_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204902fdc91b5624969b8bef9d345b810f1_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:16,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/484e214832924ddf822dbc3956575bae, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:16,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/484e214832924ddf822dbc3956575bae is 175, key is test_row_0/A:col10/1733324415049/Put/seqid=0 2024-12-04T15:00:16,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742233_1409 (size=31255) 2024-12-04T15:00:16,558 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/484e214832924ddf822dbc3956575bae 2024-12-04T15:00:16,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/50d2261ba36f46ca889107d330633ef8 is 50, key is test_row_0/B:col10/1733324415049/Put/seqid=0 2024-12-04T15:00:16,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:16,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742234_1410 (size=12301) 2024-12-04T15:00:16,605 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/50d2261ba36f46ca889107d330633ef8 2024-12-04T15:00:16,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/481932a73a864f66939370875fd97fae is 50, key is test_row_0/C:col10/1733324415049/Put/seqid=0 2024-12-04T15:00:16,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742235_1411 (size=12301) 2024-12-04T15:00:16,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:16,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324476743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:16,839 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#A#compaction#340 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:16,840 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/459909c79391433b9217963140b60086 is 175, key is test_row_0/A:col10/1733324413881/Put/seqid=0 2024-12-04T15:00:16,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742236_1412 (size=31869) 2024-12-04T15:00:16,851 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/bf69a802ee1147d08fa0a82d11b88264 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/bf69a802ee1147d08fa0a82d11b88264 2024-12-04T15:00:16,852 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/459909c79391433b9217963140b60086 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/459909c79391433b9217963140b60086 2024-12-04T15:00:16,856 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/C of e03f7dfff5f07d53203dd271e7128a0a into bf69a802ee1147d08fa0a82d11b88264(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:16,856 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:16,856 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/C, priority=12, startTime=1733324416361; duration=0sec 2024-12-04T15:00:16,856 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:16,856 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:C 2024-12-04T15:00:16,858 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/A of e03f7dfff5f07d53203dd271e7128a0a into 459909c79391433b9217963140b60086(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:16,858 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:16,858 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/A, priority=12, startTime=1733324416359; duration=0sec 2024-12-04T15:00:16,858 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:16,858 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:A 2024-12-04T15:00:16,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:16,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324476860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:16,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:16,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:16,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:17,052 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/481932a73a864f66939370875fd97fae 2024-12-04T15:00:17,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/484e214832924ddf822dbc3956575bae as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/484e214832924ddf822dbc3956575bae 2024-12-04T15:00:17,060 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/484e214832924ddf822dbc3956575bae, entries=150, sequenceid=313, filesize=30.5 K 2024-12-04T15:00:17,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/50d2261ba36f46ca889107d330633ef8 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/50d2261ba36f46ca889107d330633ef8 2024-12-04T15:00:17,064 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/50d2261ba36f46ca889107d330633ef8, entries=150, sequenceid=313, filesize=12.0 K 2024-12-04T15:00:17,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/481932a73a864f66939370875fd97fae as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/481932a73a864f66939370875fd97fae 2024-12-04T15:00:17,068 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/481932a73a864f66939370875fd97fae, entries=150, sequenceid=313, filesize=12.0 K 2024-12-04T15:00:17,069 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e03f7dfff5f07d53203dd271e7128a0a in 573ms, sequenceid=313, compaction requested=false 2024-12-04T15:00:17,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:17,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:17,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-04T15:00:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-04T15:00:17,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-04T15:00:17,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8070 sec 2024-12-04T15:00:17,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.8130 sec 2024-12-04T15:00:17,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T15:00:17,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:17,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:17,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:17,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:17,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:17,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:17,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a074286ae126413f804c9987a3460515_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324416728/Put/seqid=0 2024-12-04T15:00:17,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742237_1413 (size=14994) 2024-12-04T15:00:17,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324477144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:17,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324477232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:17,235 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in e03f7dfff5f07d53203dd271e7128a0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:17,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:17,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324477232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324477232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:17,241 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_1' in e03f7dfff5f07d53203dd271e7128a0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:17,241 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_0' in e03f7dfff5f07d53203dd271e7128a0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:17,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324477240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:17,243 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_1' in e03f7dfff5f07d53203dd271e7128a0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:17,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324477260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:17,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:17,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:17,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:17,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T15:00:17,411 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:17,411 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:00:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-04T15:00:17,413 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:00:17,414 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:00:17,414 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:00:17,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T15:00:17,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:17,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324477470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:17,500 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:17,507 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a074286ae126413f804c9987a3460515_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a074286ae126413f804c9987a3460515_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:17,515 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/b138aff8c20e41779f54a51a9193563b, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:17,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/b138aff8c20e41779f54a51a9193563b is 175, key is test_row_0/A:col10/1733324416728/Put/seqid=0 2024-12-04T15:00:17,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T15:00:17,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742238_1414 (size=39949) 2024-12-04T15:00:17,539 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/b138aff8c20e41779f54a51a9193563b 2024-12-04T15:00:17,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/0580fb20daae44a5988531b5a07a16ff is 50, key is test_row_0/B:col10/1733324416728/Put/seqid=0 2024-12-04T15:00:17,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T15:00:17,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:17,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:17,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:17,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:17,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:17,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:17,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742239_1415 (size=12301) 2024-12-04T15:00:17,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/0580fb20daae44a5988531b5a07a16ff 2024-12-04T15:00:17,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/efca068bc54343d6b87fab6bdda84fb3 is 50, key is test_row_0/C:col10/1733324416728/Put/seqid=0 2024-12-04T15:00:17,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742240_1416 (size=12301) 2024-12-04T15:00:17,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T15:00:17,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:17,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:17,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:17,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:17,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:17,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T15:00:17,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324477780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:17,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T15:00:17,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:17,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:17,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:17,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:17,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:18,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/efca068bc54343d6b87fab6bdda84fb3 2024-12-04T15:00:18,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T15:00:18,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:18,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:18,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:18,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:18,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:18,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/b138aff8c20e41779f54a51a9193563b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/b138aff8c20e41779f54a51a9193563b 2024-12-04T15:00:18,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/b138aff8c20e41779f54a51a9193563b, entries=200, sequenceid=335, filesize=39.0 K 2024-12-04T15:00:18,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/0580fb20daae44a5988531b5a07a16ff as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/0580fb20daae44a5988531b5a07a16ff 2024-12-04T15:00:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T15:00:18,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/0580fb20daae44a5988531b5a07a16ff, entries=150, sequenceid=335, filesize=12.0 K 2024-12-04T15:00:18,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/efca068bc54343d6b87fab6bdda84fb3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/efca068bc54343d6b87fab6bdda84fb3 2024-12-04T15:00:18,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/efca068bc54343d6b87fab6bdda84fb3, entries=150, sequenceid=335, filesize=12.0 K 2024-12-04T15:00:18,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e03f7dfff5f07d53203dd271e7128a0a in 964ms, sequenceid=335, compaction requested=true 2024-12-04T15:00:18,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:18,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:00:18,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:18,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:00:18,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:18,047 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:18,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:00:18,047 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:18,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:18,047 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:18,047 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:18,048 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/B is initiating minor compaction (all files) 2024-12-04T15:00:18,048 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/A is initiating minor compaction (all files) 2024-12-04T15:00:18,048 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/A in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:18,048 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/B in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:18,048 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/459909c79391433b9217963140b60086, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/484e214832924ddf822dbc3956575bae, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/b138aff8c20e41779f54a51a9193563b] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=100.7 K 2024-12-04T15:00:18,048 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/1f33b79fb0ca40c1ae7aeb6e08661845, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/50d2261ba36f46ca889107d330633ef8, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/0580fb20daae44a5988531b5a07a16ff] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=36.6 K 2024-12-04T15:00:18,048 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:18,048 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/459909c79391433b9217963140b60086, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/484e214832924ddf822dbc3956575bae, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/b138aff8c20e41779f54a51a9193563b] 2024-12-04T15:00:18,048 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f33b79fb0ca40c1ae7aeb6e08661845, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733324413880 2024-12-04T15:00:18,048 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 459909c79391433b9217963140b60086, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733324413880 2024-12-04T15:00:18,048 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 50d2261ba36f46ca889107d330633ef8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733324415044 2024-12-04T15:00:18,048 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 484e214832924ddf822dbc3956575bae, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733324415044 2024-12-04T15:00:18,048 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 0580fb20daae44a5988531b5a07a16ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733324416728 2024-12-04T15:00:18,048 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting b138aff8c20e41779f54a51a9193563b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733324416712 2024-12-04T15:00:18,053 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:18,054 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#B#compaction#349 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:18,054 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/de7409bd89934bab813cce740a8dbc61 is 50, key is test_row_0/B:col10/1733324416728/Put/seqid=0 2024-12-04T15:00:18,056 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412049dcc35483b2e46bf8f59bf175706ca26_e03f7dfff5f07d53203dd271e7128a0a store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:18,057 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412049dcc35483b2e46bf8f59bf175706ca26_e03f7dfff5f07d53203dd271e7128a0a, store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:18,057 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412049dcc35483b2e46bf8f59bf175706ca26_e03f7dfff5f07d53203dd271e7128a0a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:18,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742241_1417 (size=13017) 2024-12-04T15:00:18,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742242_1418 (size=4469) 2024-12-04T15:00:18,075 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#A#compaction#348 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:18,076 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/281a7b759b72414dbb4473fea4d297a3 is 175, key is test_row_0/A:col10/1733324416728/Put/seqid=0 2024-12-04T15:00:18,079 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/de7409bd89934bab813cce740a8dbc61 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/de7409bd89934bab813cce740a8dbc61 2024-12-04T15:00:18,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742243_1419 (size=31971) 2024-12-04T15:00:18,084 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/B of e03f7dfff5f07d53203dd271e7128a0a into de7409bd89934bab813cce740a8dbc61(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:18,084 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:18,084 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/B, priority=13, startTime=1733324418047; duration=0sec 2024-12-04T15:00:18,084 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:18,084 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:B 2024-12-04T15:00:18,084 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:18,088 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:18,088 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/C is initiating minor compaction (all files) 2024-12-04T15:00:18,088 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/C in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:18,088 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/bf69a802ee1147d08fa0a82d11b88264, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/481932a73a864f66939370875fd97fae, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/efca068bc54343d6b87fab6bdda84fb3] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=36.6 K 2024-12-04T15:00:18,089 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting bf69a802ee1147d08fa0a82d11b88264, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733324413880 2024-12-04T15:00:18,091 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 481932a73a864f66939370875fd97fae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733324415044 2024-12-04T15:00:18,091 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/281a7b759b72414dbb4473fea4d297a3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/281a7b759b72414dbb4473fea4d297a3 2024-12-04T15:00:18,097 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/A of e03f7dfff5f07d53203dd271e7128a0a into 281a7b759b72414dbb4473fea4d297a3(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:18,097 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:18,097 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/A, priority=13, startTime=1733324418047; duration=0sec 2024-12-04T15:00:18,097 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:18,097 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:A 2024-12-04T15:00:18,102 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting efca068bc54343d6b87fab6bdda84fb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733324416728 2024-12-04T15:00:18,108 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#C#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:18,108 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/9539282decd04bd6b1c413a9dcbf1e78 is 50, key is test_row_0/C:col10/1733324416728/Put/seqid=0 2024-12-04T15:00:18,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742244_1420 (size=13017) 2024-12-04T15:00:18,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T15:00:18,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:18,182 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T15:00:18,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:18,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:18,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:18,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:18,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:18,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:18,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040f80cb3dd5e144fc92934c01d5fdf414_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324417126/Put/seqid=0 2024-12-04T15:00:18,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742245_1421 (size=12454) 2024-12-04T15:00:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:18,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:18,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:18,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324478441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:18,526 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/9539282decd04bd6b1c413a9dcbf1e78 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/9539282decd04bd6b1c413a9dcbf1e78 2024-12-04T15:00:18,536 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/C of e03f7dfff5f07d53203dd271e7128a0a into 9539282decd04bd6b1c413a9dcbf1e78(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:18,536 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:18,536 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/C, priority=13, startTime=1733324418047; duration=0sec 2024-12-04T15:00:18,536 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:18,536 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:C 2024-12-04T15:00:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T15:00:18,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324478550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:18,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:18,606 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040f80cb3dd5e144fc92934c01d5fdf414_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040f80cb3dd5e144fc92934c01d5fdf414_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:18,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/dbd9dc0401b243a2a9019617877bf57a, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:18,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/dbd9dc0401b243a2a9019617877bf57a is 175, key is test_row_0/A:col10/1733324417126/Put/seqid=0 2024-12-04T15:00:18,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742246_1422 (size=31255) 2024-12-04T15:00:18,614 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/dbd9dc0401b243a2a9019617877bf57a 2024-12-04T15:00:18,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/be3bdcd1f33f4a0fb4300e9ba7718566 is 50, key is test_row_0/B:col10/1733324417126/Put/seqid=0 2024-12-04T15:00:18,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742247_1423 (size=12301) 2024-12-04T15:00:18,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324478770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:18,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:18,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:18,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:19,037 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/be3bdcd1f33f4a0fb4300e9ba7718566 2024-12-04T15:00:19,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/3fe2211dda284d3dbbaba5c68d6c1456 is 50, key is test_row_0/C:col10/1733324417126/Put/seqid=0 2024-12-04T15:00:19,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742248_1424 (size=12301) 2024-12-04T15:00:19,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324479081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:19,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:19,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:19,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324479250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:19,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:19,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324479251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:19,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:19,257 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in e03f7dfff5f07d53203dd271e7128a0a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:19,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324479251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:19,258 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_0' in e03f7dfff5f07d53203dd271e7128a0a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:19,259 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_1' in e03f7dfff5f07d53203dd271e7128a0a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:19,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:19,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:19,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:19,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324479260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:19,270 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_1' in e03f7dfff5f07d53203dd271e7128a0a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:19,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:19,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:19,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:19,468 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/3fe2211dda284d3dbbaba5c68d6c1456 2024-12-04T15:00:19,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/dbd9dc0401b243a2a9019617877bf57a as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/dbd9dc0401b243a2a9019617877bf57a 2024-12-04T15:00:19,478 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/dbd9dc0401b243a2a9019617877bf57a, entries=150, sequenceid=349, filesize=30.5 K 2024-12-04T15:00:19,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/be3bdcd1f33f4a0fb4300e9ba7718566 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/be3bdcd1f33f4a0fb4300e9ba7718566 2024-12-04T15:00:19,483 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/be3bdcd1f33f4a0fb4300e9ba7718566, entries=150, sequenceid=349, filesize=12.0 K 2024-12-04T15:00:19,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/3fe2211dda284d3dbbaba5c68d6c1456 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/3fe2211dda284d3dbbaba5c68d6c1456 2024-12-04T15:00:19,487 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/3fe2211dda284d3dbbaba5c68d6c1456, entries=150, sequenceid=349, filesize=12.0 K 2024-12-04T15:00:19,488 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e03f7dfff5f07d53203dd271e7128a0a in 1306ms, sequenceid=349, compaction requested=false 2024-12-04T15:00:19,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:19,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:19,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-04T15:00:19,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-04T15:00:19,493 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-04T15:00:19,493 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0770 sec 2024-12-04T15:00:19,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.0860 sec 2024-12-04T15:00:19,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T15:00:19,560 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:19,561 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:00:19,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-04T15:00:19,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T15:00:19,562 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:00:19,563 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:00:19,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:00:19,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:19,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-04T15:00:19,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:19,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:19,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:19,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:19,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:19,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:19,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040272d2e5d7f04c19a42144758a5bd12b_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324418431/Put/seqid=0 2024-12-04T15:00:19,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742249_1425 (size=14994) 2024-12-04T15:00:19,612 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:19,616 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040272d2e5d7f04c19a42144758a5bd12b_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040272d2e5d7f04c19a42144758a5bd12b_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:19,618 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/c4722604fd204a05bfb8f1e1daa625f2, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:19,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/c4722604fd204a05bfb8f1e1daa625f2 is 175, key is test_row_0/A:col10/1733324418431/Put/seqid=0 2024-12-04T15:00:19,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742250_1426 (size=39949) 2024-12-04T15:00:19,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:19,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324479643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:19,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T15:00:19,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T15:00:19,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:19,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:19,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:19,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:19,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:19,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:19,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:19,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324479750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:19,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:19,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:19,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:19,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T15:00:19,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:19,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:19,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:19,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:19,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:19,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:19,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T15:00:19,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:19,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324479961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:19,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:19,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:19,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:20,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T15:00:20,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:20,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:20,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:20,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:20,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:20,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:20,023 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=371, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/c4722604fd204a05bfb8f1e1daa625f2 2024-12-04T15:00:20,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/1af0459e7aad488ebe293fd0cdf2994e is 50, key is test_row_0/B:col10/1733324418431/Put/seqid=0 2024-12-04T15:00:20,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742251_1427 (size=12301) 2024-12-04T15:00:20,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T15:00:20,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:20,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:20,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:20,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:20,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:20,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:20,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T15:00:20,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:20,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324480270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:20,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T15:00:20,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:20,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:20,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:20,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:20,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:20,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/1af0459e7aad488ebe293fd0cdf2994e 2024-12-04T15:00:20,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/5d250ea5eb5b482788d2ed166e38cda0 is 50, key is test_row_0/C:col10/1733324418431/Put/seqid=0 2024-12-04T15:00:20,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742252_1428 (size=12301) 2024-12-04T15:00:20,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T15:00:20,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:20,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:20,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:20,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:20,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:20,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T15:00:20,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:20,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:20,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:20,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:20,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:20,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T15:00:20,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:20,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324480780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:20,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T15:00:20,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:20,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:20,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:20,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:20,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:20,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/5d250ea5eb5b482788d2ed166e38cda0 2024-12-04T15:00:20,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/c4722604fd204a05bfb8f1e1daa625f2 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/c4722604fd204a05bfb8f1e1daa625f2 2024-12-04T15:00:20,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/c4722604fd204a05bfb8f1e1daa625f2, entries=200, sequenceid=371, filesize=39.0 K 2024-12-04T15:00:20,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/1af0459e7aad488ebe293fd0cdf2994e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/1af0459e7aad488ebe293fd0cdf2994e 2024-12-04T15:00:20,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/1af0459e7aad488ebe293fd0cdf2994e, entries=150, sequenceid=371, filesize=12.0 K 2024-12-04T15:00:20,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/5d250ea5eb5b482788d2ed166e38cda0 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/5d250ea5eb5b482788d2ed166e38cda0 2024-12-04T15:00:20,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/5d250ea5eb5b482788d2ed166e38cda0, entries=150, sequenceid=371, filesize=12.0 K 2024-12-04T15:00:20,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for e03f7dfff5f07d53203dd271e7128a0a in 1272ms, sequenceid=371, compaction requested=true 2024-12-04T15:00:20,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:20,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:00:20,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:20,867 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:20,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:00:20,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:20,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:00:20,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:20,867 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:20,868 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:20,868 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/B is initiating minor compaction (all files) 2024-12-04T15:00:20,869 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/B in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:20,869 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/de7409bd89934bab813cce740a8dbc61, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/be3bdcd1f33f4a0fb4300e9ba7718566, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/1af0459e7aad488ebe293fd0cdf2994e] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=36.7 K 2024-12-04T15:00:20,869 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:20,869 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/A is initiating minor compaction (all files) 2024-12-04T15:00:20,869 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/A in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:20,869 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/281a7b759b72414dbb4473fea4d297a3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/dbd9dc0401b243a2a9019617877bf57a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/c4722604fd204a05bfb8f1e1daa625f2] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=100.8 K 2024-12-04T15:00:20,869 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:20,869 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/281a7b759b72414dbb4473fea4d297a3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/dbd9dc0401b243a2a9019617877bf57a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/c4722604fd204a05bfb8f1e1daa625f2] 2024-12-04T15:00:20,870 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting de7409bd89934bab813cce740a8dbc61, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733324416728 2024-12-04T15:00:20,870 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 281a7b759b72414dbb4473fea4d297a3, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733324416728 2024-12-04T15:00:20,870 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting be3bdcd1f33f4a0fb4300e9ba7718566, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733324417126 2024-12-04T15:00:20,870 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbd9dc0401b243a2a9019617877bf57a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733324417126 2024-12-04T15:00:20,870 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1af0459e7aad488ebe293fd0cdf2994e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733324418422 2024-12-04T15:00:20,871 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4722604fd204a05bfb8f1e1daa625f2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733324418422 2024-12-04T15:00:20,879 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:20,881 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#B#compaction#358 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:20,882 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/a5449ad2ac564829a88ec8400ec3eee4 is 50, key is test_row_0/B:col10/1733324418431/Put/seqid=0 2024-12-04T15:00:20,889 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412041cc5bd97bb7a4b0cbf0cd6d104cef314_e03f7dfff5f07d53203dd271e7128a0a store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:20,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742253_1429 (size=13119) 2024-12-04T15:00:20,891 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412041cc5bd97bb7a4b0cbf0cd6d104cef314_e03f7dfff5f07d53203dd271e7128a0a, store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:20,891 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041cc5bd97bb7a4b0cbf0cd6d104cef314_e03f7dfff5f07d53203dd271e7128a0a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:20,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742254_1430 (size=4469) 2024-12-04T15:00:20,902 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/a5449ad2ac564829a88ec8400ec3eee4 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/a5449ad2ac564829a88ec8400ec3eee4 2024-12-04T15:00:20,908 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/B of e03f7dfff5f07d53203dd271e7128a0a into a5449ad2ac564829a88ec8400ec3eee4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:20,908 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:20,908 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/B, priority=13, startTime=1733324420867; duration=0sec 2024-12-04T15:00:20,909 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:20,909 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:B 2024-12-04T15:00:20,909 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:20,910 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#A#compaction#357 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:20,910 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/1176215428354380a4c4fe578c011866 is 175, key is test_row_0/A:col10/1733324418431/Put/seqid=0 2024-12-04T15:00:20,910 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:20,911 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/C is initiating minor compaction (all files) 2024-12-04T15:00:20,911 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/C in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:20,911 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/9539282decd04bd6b1c413a9dcbf1e78, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/3fe2211dda284d3dbbaba5c68d6c1456, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/5d250ea5eb5b482788d2ed166e38cda0] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=36.7 K 2024-12-04T15:00:20,911 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 9539282decd04bd6b1c413a9dcbf1e78, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733324416728 2024-12-04T15:00:20,911 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fe2211dda284d3dbbaba5c68d6c1456, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733324417126 2024-12-04T15:00:20,912 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d250ea5eb5b482788d2ed166e38cda0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733324418422 2024-12-04T15:00:20,921 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#C#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:20,921 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/db49c109d6e64cbe934499367f75c445 is 50, key is test_row_0/C:col10/1733324418431/Put/seqid=0 2024-12-04T15:00:20,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742255_1431 (size=32073) 2024-12-04T15:00:20,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T15:00:20,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:20,941 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T15:00:20,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:20,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:20,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:20,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:20,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:20,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:20,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742256_1432 (size=13119) 2024-12-04T15:00:20,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204da7b4651c0594b2abedd25886bdcb8dd_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324419632/Put/seqid=0 2024-12-04T15:00:20,961 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/db49c109d6e64cbe934499367f75c445 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/db49c109d6e64cbe934499367f75c445 2024-12-04T15:00:20,967 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/C of e03f7dfff5f07d53203dd271e7128a0a into db49c109d6e64cbe934499367f75c445(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:20,967 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:20,967 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/C, priority=13, startTime=1733324420867; duration=0sec 2024-12-04T15:00:20,967 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:20,967 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:C 2024-12-04T15:00:20,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742257_1433 (size=12454) 2024-12-04T15:00:21,329 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/1176215428354380a4c4fe578c011866 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/1176215428354380a4c4fe578c011866 2024-12-04T15:00:21,333 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/A of e03f7dfff5f07d53203dd271e7128a0a into 1176215428354380a4c4fe578c011866(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:21,334 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:21,334 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/A, priority=13, startTime=1733324420866; duration=0sec 2024-12-04T15:00:21,334 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:21,334 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:A 2024-12-04T15:00:21,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:21,379 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204da7b4651c0594b2abedd25886bdcb8dd_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204da7b4651c0594b2abedd25886bdcb8dd_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:21,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/139b447a7b0246a2a3091f620fe0a29e, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:21,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/139b447a7b0246a2a3091f620fe0a29e is 175, key is test_row_0/A:col10/1733324419632/Put/seqid=0 2024-12-04T15:00:21,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742258_1434 (size=31255) 2024-12-04T15:00:21,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T15:00:21,784 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=383, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/139b447a7b0246a2a3091f620fe0a29e 2024-12-04T15:00:21,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/74804e4d808d430c86a62bb9d572d811 is 50, key is test_row_0/B:col10/1733324419632/Put/seqid=0 2024-12-04T15:00:21,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742259_1435 (size=12301) 2024-12-04T15:00:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:21,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. as already flushing 2024-12-04T15:00:21,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324421847, max timestamp=1733324421885Num uniques 859; 2024-12-04T15:00:21,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T15:00:21,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324481928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:21,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:21,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:21,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:22,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324482040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:22,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:22,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:22,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:22,192 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/74804e4d808d430c86a62bb9d572d811 2024-12-04T15:00:22,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/0a2c3e1cabdc424bb5606b62a06a1342 is 50, key is test_row_0/C:col10/1733324419632/Put/seqid=0 2024-12-04T15:00:22,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742260_1436 (size=12301) 2024-12-04T15:00:22,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324482250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:22,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:22,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:22,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:22,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324482560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:22,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:22,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:22,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:22,600 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/0a2c3e1cabdc424bb5606b62a06a1342 2024-12-04T15:00:22,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/139b447a7b0246a2a3091f620fe0a29e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/139b447a7b0246a2a3091f620fe0a29e 2024-12-04T15:00:22,609 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/139b447a7b0246a2a3091f620fe0a29e, entries=150, sequenceid=383, filesize=30.5 K 2024-12-04T15:00:22,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/74804e4d808d430c86a62bb9d572d811 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/74804e4d808d430c86a62bb9d572d811 2024-12-04T15:00:22,612 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/74804e4d808d430c86a62bb9d572d811, entries=150, sequenceid=383, filesize=12.0 K 2024-12-04T15:00:22,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/0a2c3e1cabdc424bb5606b62a06a1342 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/0a2c3e1cabdc424bb5606b62a06a1342 2024-12-04T15:00:22,616 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/0a2c3e1cabdc424bb5606b62a06a1342, entries=150, sequenceid=383, filesize=12.0 K 2024-12-04T15:00:22,617 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=123.00 KB/125950 for e03f7dfff5f07d53203dd271e7128a0a in 1675ms, sequenceid=383, compaction requested=false 2024-12-04T15:00:22,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:22,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:22,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-04T15:00:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-04T15:00:22,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-04T15:00:22,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0550 sec 2024-12-04T15:00:22,621 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 3.0590 sec 2024-12-04T15:00:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:23,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=129.71 KB heapSize=381.64 KB 2024-12-04T15:00:23,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:23,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:23,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:23,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:23,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:23,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:23,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412042244aec9df674db5a2e9cb417be21f41_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324421910/Put/seqid=0 2024-12-04T15:00:23,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742261_1437 (size=14994) 2024-12-04T15:00:23,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324483115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:23,226 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T15:00:23,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324483230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:23,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1733324483270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:23,272 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_1' in e03f7dfff5f07d53203dd271e7128a0a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:23,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:23,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:23,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:23,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53812 deadline: 1733324483280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:23,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53880 deadline: 1733324483280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:23,286 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_0' in e03f7dfff5f07d53203dd271e7128a0a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:23,287 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_1' in e03f7dfff5f07d53203dd271e7128a0a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:23,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1733324483290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:23,296 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in e03f7dfff5f07d53203dd271e7128a0a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:23,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:23,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:23,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:23,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53834 deadline: 1733324483440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:23,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:23,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:00:23,484 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:23,487 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412042244aec9df674db5a2e9cb417be21f41_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042244aec9df674db5a2e9cb417be21f41_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:23,488 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/1294fee996b9439f9f5139170f455a2f, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:23,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/1294fee996b9439f9f5139170f455a2f is 175, key is test_row_0/A:col10/1733324421910/Put/seqid=0 2024-12-04T15:00:23,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742262_1438 (size=39949) 2024-12-04T15:00:23,499 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=412, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/1294fee996b9439f9f5139170f455a2f 2024-12-04T15:00:23,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/fe538ebbda6b4d48bd024e8fcb21157c is 50, key is test_row_0/B:col10/1733324421910/Put/seqid=0 2024-12-04T15:00:23,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742263_1439 (size=12301) 2024-12-04T15:00:23,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/fe538ebbda6b4d48bd024e8fcb21157c 2024-12-04T15:00:23,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/c4cf383fff90482b9418eaa1f8a8a2de is 50, key is test_row_0/C:col10/1733324421910/Put/seqid=0 2024-12-04T15:00:23,515 INFO [Thread-1536 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1536. 2024-12-04T15:00:23,515 DEBUG [Thread-1536 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:00:23,515 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:23,515 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:23,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:00:23,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742264_1440 (size=12301) 2024-12-04T15:00:23,517 INFO [Thread-1534 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1534. 2024-12-04T15:00:23,517 DEBUG [Thread-1534 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:00:23,517 DEBUG [Thread-1534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:23,517 DEBUG [Thread-1534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:23,517 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:00:23,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/c4cf383fff90482b9418eaa1f8a8a2de 2024-12-04T15:00:23,519 INFO [Thread-1538 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1538. 2024-12-04T15:00:23,519 DEBUG [Thread-1538 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:00:23,519 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:23,519 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:23,519 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:00:23,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/1294fee996b9439f9f5139170f455a2f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/1294fee996b9439f9f5139170f455a2f 2024-12-04T15:00:23,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/1294fee996b9439f9f5139170f455a2f, entries=200, sequenceid=412, filesize=39.0 K 2024-12-04T15:00:23,524 INFO [Thread-1530 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1530. 2024-12-04T15:00:23,524 DEBUG [Thread-1530 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:00:23,524 DEBUG [Thread-1530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:23,525 DEBUG [Thread-1530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:23,525 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:00:23,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/fe538ebbda6b4d48bd024e8fcb21157c as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/fe538ebbda6b4d48bd024e8fcb21157c 2024-12-04T15:00:23,526 INFO [Thread-1532 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1532. 2024-12-04T15:00:23,526 DEBUG [Thread-1532 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:00:23,526 DEBUG [Thread-1532 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:23,526 DEBUG [Thread-1532 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:23,526 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:00:23,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/fe538ebbda6b4d48bd024e8fcb21157c, entries=150, sequenceid=412, filesize=12.0 K 2024-12-04T15:00:23,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/c4cf383fff90482b9418eaa1f8a8a2de as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/c4cf383fff90482b9418eaa1f8a8a2de 2024-12-04T15:00:23,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/c4cf383fff90482b9418eaa1f8a8a2de, entries=150, sequenceid=412, filesize=12.0 K 2024-12-04T15:00:23,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~381.59 KB/390752, currentSize=46.96 KB/48090 for e03f7dfff5f07d53203dd271e7128a0a in 457ms, sequenceid=412, compaction requested=true 2024-12-04T15:00:23,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:23,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:00:23,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:23,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:00:23,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:23,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e03f7dfff5f07d53203dd271e7128a0a:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:00:23,531 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:23,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:23,531 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:23,532 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:23,532 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:23,532 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/B is initiating minor compaction (all files) 2024-12-04T15:00:23,532 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/A is initiating minor compaction (all files) 2024-12-04T15:00:23,532 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/A in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:23,532 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/B in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:23,532 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/a5449ad2ac564829a88ec8400ec3eee4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/74804e4d808d430c86a62bb9d572d811, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/fe538ebbda6b4d48bd024e8fcb21157c] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=36.8 K 2024-12-04T15:00:23,532 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/1176215428354380a4c4fe578c011866, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/139b447a7b0246a2a3091f620fe0a29e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/1294fee996b9439f9f5139170f455a2f] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=100.9 K 2024-12-04T15:00:23,532 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:23,532 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/1176215428354380a4c4fe578c011866, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/139b447a7b0246a2a3091f620fe0a29e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/1294fee996b9439f9f5139170f455a2f] 2024-12-04T15:00:23,532 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting a5449ad2ac564829a88ec8400ec3eee4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733324418422 2024-12-04T15:00:23,533 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1176215428354380a4c4fe578c011866, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733324418422 2024-12-04T15:00:23,533 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 74804e4d808d430c86a62bb9d572d811, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733324419617 2024-12-04T15:00:23,533 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 139b447a7b0246a2a3091f620fe0a29e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733324419617 2024-12-04T15:00:23,533 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting fe538ebbda6b4d48bd024e8fcb21157c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733324421910 2024-12-04T15:00:23,533 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1294fee996b9439f9f5139170f455a2f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733324421910 2024-12-04T15:00:23,538 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#B#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:23,539 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/aafefc8816004690851b9f2ee06e70a7 is 50, key is test_row_0/B:col10/1733324421910/Put/seqid=0 2024-12-04T15:00:23,541 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:23,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742265_1441 (size=13221) 2024-12-04T15:00:23,547 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204ab1da3cf35ac4458b3b2665354f9bf7f_e03f7dfff5f07d53203dd271e7128a0a store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:23,562 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204ab1da3cf35ac4458b3b2665354f9bf7f_e03f7dfff5f07d53203dd271e7128a0a, store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:23,562 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204ab1da3cf35ac4458b3b2665354f9bf7f_e03f7dfff5f07d53203dd271e7128a0a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:23,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742266_1442 (size=4469) 2024-12-04T15:00:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T15:00:23,731 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:23,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T15:00:23,751 INFO [Thread-1521 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1521. 2024-12-04T15:00:23,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:23,751 DEBUG [Thread-1521 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:00:23,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:23,751 DEBUG [Thread-1521 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:23,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:23,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:23,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:23,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:23,751 DEBUG [Thread-1521 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:23,751 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:00:23,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041783a8301f02463ba6011c9c5392f98c_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324423750/Put/seqid=0 2024-12-04T15:00:23,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742267_1443 (size=12454) 2024-12-04T15:00:23,951 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/aafefc8816004690851b9f2ee06e70a7 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/aafefc8816004690851b9f2ee06e70a7 2024-12-04T15:00:23,959 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/B of e03f7dfff5f07d53203dd271e7128a0a into aafefc8816004690851b9f2ee06e70a7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:23,959 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:23,959 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/B, priority=13, startTime=1733324423531; duration=0sec 2024-12-04T15:00:23,959 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:23,959 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:B 2024-12-04T15:00:23,959 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:23,960 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:23,960 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): e03f7dfff5f07d53203dd271e7128a0a/C is initiating minor compaction (all files) 2024-12-04T15:00:23,960 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e03f7dfff5f07d53203dd271e7128a0a/C in TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:23,960 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/db49c109d6e64cbe934499367f75c445, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/0a2c3e1cabdc424bb5606b62a06a1342, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/c4cf383fff90482b9418eaa1f8a8a2de] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp, totalSize=36.8 K 2024-12-04T15:00:23,961 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting db49c109d6e64cbe934499367f75c445, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733324418422 2024-12-04T15:00:23,961 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a2c3e1cabdc424bb5606b62a06a1342, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733324419617 2024-12-04T15:00:23,961 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting c4cf383fff90482b9418eaa1f8a8a2de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733324421910 2024-12-04T15:00:23,966 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#A#compaction#367 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:23,967 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/b37c0d2473d3433daee5f4ab2fcb397a is 175, key is test_row_0/A:col10/1733324421910/Put/seqid=0 2024-12-04T15:00:23,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742268_1444 (size=32175) 2024-12-04T15:00:23,972 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e03f7dfff5f07d53203dd271e7128a0a#C#compaction#369 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:23,973 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/f5c4de57c47b430aac24d950a7248add is 50, key is test_row_0/C:col10/1733324421910/Put/seqid=0 2024-12-04T15:00:23,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742269_1445 (size=13221) 2024-12-04T15:00:24,161 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:24,165 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041783a8301f02463ba6011c9c5392f98c_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041783a8301f02463ba6011c9c5392f98c_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:24,166 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/ac26cdcf942346c58517a40452602c6a, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:24,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/ac26cdcf942346c58517a40452602c6a is 175, key is test_row_0/A:col10/1733324423750/Put/seqid=0 2024-12-04T15:00:24,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742270_1446 (size=31255) 2024-12-04T15:00:24,379 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/b37c0d2473d3433daee5f4ab2fcb397a as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/b37c0d2473d3433daee5f4ab2fcb397a 2024-12-04T15:00:24,382 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/f5c4de57c47b430aac24d950a7248add as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/f5c4de57c47b430aac24d950a7248add 2024-12-04T15:00:24,386 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/A of e03f7dfff5f07d53203dd271e7128a0a into b37c0d2473d3433daee5f4ab2fcb397a(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:24,386 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:24,386 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/A, priority=13, startTime=1733324423531; duration=0sec 2024-12-04T15:00:24,386 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:24,386 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:A 2024-12-04T15:00:24,388 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e03f7dfff5f07d53203dd271e7128a0a/C of e03f7dfff5f07d53203dd271e7128a0a into f5c4de57c47b430aac24d950a7248add(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:24,388 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:24,388 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., storeName=e03f7dfff5f07d53203dd271e7128a0a/C, priority=13, startTime=1733324423531; duration=0sec 2024-12-04T15:00:24,388 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:24,388 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e03f7dfff5f07d53203dd271e7128a0a:C 2024-12-04T15:00:24,572 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=423, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/ac26cdcf942346c58517a40452602c6a 2024-12-04T15:00:24,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/633755811af14da69550b02518d820a3 is 50, key is test_row_0/B:col10/1733324423750/Put/seqid=0 2024-12-04T15:00:24,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742271_1447 (size=12301) 2024-12-04T15:00:24,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/633755811af14da69550b02518d820a3 2024-12-04T15:00:25,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/cd59eedc936042dca4699fc9e0c4563e is 50, key is test_row_0/C:col10/1733324423750/Put/seqid=0 2024-12-04T15:00:25,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742272_1448 (size=12301) 2024-12-04T15:00:25,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/cd59eedc936042dca4699fc9e0c4563e 2024-12-04T15:00:25,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/ac26cdcf942346c58517a40452602c6a as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/ac26cdcf942346c58517a40452602c6a 2024-12-04T15:00:25,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/ac26cdcf942346c58517a40452602c6a, entries=150, sequenceid=423, filesize=30.5 K 2024-12-04T15:00:25,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/633755811af14da69550b02518d820a3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/633755811af14da69550b02518d820a3 2024-12-04T15:00:25,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/633755811af14da69550b02518d820a3, entries=150, sequenceid=423, filesize=12.0 K 2024-12-04T15:00:25,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/cd59eedc936042dca4699fc9e0c4563e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/cd59eedc936042dca4699fc9e0c4563e 2024-12-04T15:00:25,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/cd59eedc936042dca4699fc9e0c4563e, entries=150, sequenceid=423, filesize=12.0 K 2024-12-04T15:00:25,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for e03f7dfff5f07d53203dd271e7128a0a in 1282ms, sequenceid=423, compaction requested=false 2024-12-04T15:00:25,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e03f7dfff5f07d53203dd271e7128a0a: 2024-12-04T15:00:27,320 WARN [regionserver/d214edae7bd2:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-04T15:00:27,323 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/59e4b30bb40847f7a11c7c316bbac777, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/c5301bf669e54845837140cf12128c3e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/74eb0e5c11334c34aa3e4b62d9d2e0c4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/e0af19c292c04778995adabf3ec6499b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/44cd52a3414b465694e5949121fa40d0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/f380f889c06f4852b3bbcb6d513dc7e0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/5e18309a541b4be788a54176f82366e7, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/66b9bc1632334fbf890b2b56e79e30ca, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/84a31991bc9b438397989aca86c30b1c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/dd0b10c047dc4b95ada36f66d2242d67, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/ac231970ee5e472a8dc79ae6680675a0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/07e6089445ed49d68166342b94571279, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/26ce439201e94de3afcf30af94c49815, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/997868d0a90847109c5eaa2672ab99a6, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/06b08331f1a047fdae121860012cee4a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/da702713cc3e45c7805b0f6b143a66d1, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/e6220b8ffa224ea99e237f7ef5bd5b68, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/ba1020932bad4f56ab6458147fc08661, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/459909c79391433b9217963140b60086, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/484e214832924ddf822dbc3956575bae, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/b138aff8c20e41779f54a51a9193563b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/281a7b759b72414dbb4473fea4d297a3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/dbd9dc0401b243a2a9019617877bf57a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/c4722604fd204a05bfb8f1e1daa625f2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/1176215428354380a4c4fe578c011866, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/139b447a7b0246a2a3091f620fe0a29e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/1294fee996b9439f9f5139170f455a2f] to archive 2024-12-04T15:00:27,325 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T15:00:27,390 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/59e4b30bb40847f7a11c7c316bbac777 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/59e4b30bb40847f7a11c7c316bbac777 2024-12-04T15:00:27,392 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/c5301bf669e54845837140cf12128c3e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/c5301bf669e54845837140cf12128c3e 2024-12-04T15:00:27,394 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/74eb0e5c11334c34aa3e4b62d9d2e0c4 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/74eb0e5c11334c34aa3e4b62d9d2e0c4 2024-12-04T15:00:27,396 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/e0af19c292c04778995adabf3ec6499b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/e0af19c292c04778995adabf3ec6499b 2024-12-04T15:00:27,397 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/44cd52a3414b465694e5949121fa40d0 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/44cd52a3414b465694e5949121fa40d0 2024-12-04T15:00:27,399 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/f380f889c06f4852b3bbcb6d513dc7e0 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/f380f889c06f4852b3bbcb6d513dc7e0 2024-12-04T15:00:27,415 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/5e18309a541b4be788a54176f82366e7 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/5e18309a541b4be788a54176f82366e7 2024-12-04T15:00:27,416 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/66b9bc1632334fbf890b2b56e79e30ca to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/66b9bc1632334fbf890b2b56e79e30ca 2024-12-04T15:00:27,417 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/84a31991bc9b438397989aca86c30b1c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/84a31991bc9b438397989aca86c30b1c 2024-12-04T15:00:27,418 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/dd0b10c047dc4b95ada36f66d2242d67 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/dd0b10c047dc4b95ada36f66d2242d67 2024-12-04T15:00:27,423 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/ac231970ee5e472a8dc79ae6680675a0 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/ac231970ee5e472a8dc79ae6680675a0 2024-12-04T15:00:27,427 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/07e6089445ed49d68166342b94571279 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/07e6089445ed49d68166342b94571279 2024-12-04T15:00:27,430 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/26ce439201e94de3afcf30af94c49815 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/26ce439201e94de3afcf30af94c49815 2024-12-04T15:00:27,433 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/997868d0a90847109c5eaa2672ab99a6 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/997868d0a90847109c5eaa2672ab99a6 2024-12-04T15:00:27,436 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/06b08331f1a047fdae121860012cee4a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/06b08331f1a047fdae121860012cee4a 2024-12-04T15:00:27,437 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/da702713cc3e45c7805b0f6b143a66d1 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/da702713cc3e45c7805b0f6b143a66d1 2024-12-04T15:00:27,501 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/e6220b8ffa224ea99e237f7ef5bd5b68 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/e6220b8ffa224ea99e237f7ef5bd5b68 2024-12-04T15:00:27,502 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/ba1020932bad4f56ab6458147fc08661 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/ba1020932bad4f56ab6458147fc08661 2024-12-04T15:00:27,505 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/459909c79391433b9217963140b60086 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/459909c79391433b9217963140b60086 2024-12-04T15:00:27,517 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/484e214832924ddf822dbc3956575bae to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/484e214832924ddf822dbc3956575bae 2024-12-04T15:00:27,523 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/b138aff8c20e41779f54a51a9193563b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/b138aff8c20e41779f54a51a9193563b 2024-12-04T15:00:27,554 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/281a7b759b72414dbb4473fea4d297a3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/281a7b759b72414dbb4473fea4d297a3 2024-12-04T15:00:27,568 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/dbd9dc0401b243a2a9019617877bf57a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/dbd9dc0401b243a2a9019617877bf57a 2024-12-04T15:00:27,580 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/c4722604fd204a05bfb8f1e1daa625f2 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/c4722604fd204a05bfb8f1e1daa625f2 2024-12-04T15:00:27,581 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/1176215428354380a4c4fe578c011866 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/1176215428354380a4c4fe578c011866 2024-12-04T15:00:27,582 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/139b447a7b0246a2a3091f620fe0a29e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/139b447a7b0246a2a3091f620fe0a29e 2024-12-04T15:00:27,585 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/1294fee996b9439f9f5139170f455a2f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/1294fee996b9439f9f5139170f455a2f 2024-12-04T15:00:27,661 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c5323683d3d24a319a119689c08e4a25, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/099360fef94247859824b36e05b277e5, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/8b2749fc7dba4ef7a1353c99f605f074, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c11c881a32284321ad3066a4c5c61e48, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c35fb5dbda3a48438bfd11322e0f9dcf, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/ec6e9f5514b64004afa877852f9145a6, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/1d3506c563734ddd81f337878ee85f6e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/2399476653d341eaa7211f612b20b284, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/2dd651dccc934cc8964fe1d7c8a11b26, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/a3f2d38d57de40a78b67460f7d5200a8, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/14faa8bbe90d48e186eadf451128bf2b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/9f55cb45fa3a45c892ed89750b852f1b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/148e5a59aa0642c489ea33cad4750975, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/273be2c0db2f478590487f934a9d149f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/8be9cb19ef804fd2a39ce2434421a8f7, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/4250f3f1782a4156a421321bd4e88eb3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/7f2a660307fa40759b0433e847fb89ad, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/1f33b79fb0ca40c1ae7aeb6e08661845, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/b6ecf3f70d51460db498888a1b6c9ee4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/50d2261ba36f46ca889107d330633ef8, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/de7409bd89934bab813cce740a8dbc61, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/0580fb20daae44a5988531b5a07a16ff, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/be3bdcd1f33f4a0fb4300e9ba7718566, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/a5449ad2ac564829a88ec8400ec3eee4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/1af0459e7aad488ebe293fd0cdf2994e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/74804e4d808d430c86a62bb9d572d811, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/fe538ebbda6b4d48bd024e8fcb21157c] to archive 2024-12-04T15:00:27,662 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T15:00:27,663 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c5323683d3d24a319a119689c08e4a25 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c5323683d3d24a319a119689c08e4a25 2024-12-04T15:00:27,665 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/099360fef94247859824b36e05b277e5 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/099360fef94247859824b36e05b277e5 2024-12-04T15:00:27,667 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/8b2749fc7dba4ef7a1353c99f605f074 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/8b2749fc7dba4ef7a1353c99f605f074 2024-12-04T15:00:27,669 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c11c881a32284321ad3066a4c5c61e48 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c11c881a32284321ad3066a4c5c61e48 2024-12-04T15:00:27,670 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c35fb5dbda3a48438bfd11322e0f9dcf to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c35fb5dbda3a48438bfd11322e0f9dcf 2024-12-04T15:00:27,701 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/ec6e9f5514b64004afa877852f9145a6 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/ec6e9f5514b64004afa877852f9145a6 2024-12-04T15:00:27,712 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/1d3506c563734ddd81f337878ee85f6e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/1d3506c563734ddd81f337878ee85f6e 2024-12-04T15:00:27,722 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/2399476653d341eaa7211f612b20b284 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/2399476653d341eaa7211f612b20b284 2024-12-04T15:00:27,732 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/2dd651dccc934cc8964fe1d7c8a11b26 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/2dd651dccc934cc8964fe1d7c8a11b26 2024-12-04T15:00:27,740 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/a3f2d38d57de40a78b67460f7d5200a8 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/a3f2d38d57de40a78b67460f7d5200a8 2024-12-04T15:00:27,741 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/14faa8bbe90d48e186eadf451128bf2b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/14faa8bbe90d48e186eadf451128bf2b 2024-12-04T15:00:27,742 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/9f55cb45fa3a45c892ed89750b852f1b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/9f55cb45fa3a45c892ed89750b852f1b 2024-12-04T15:00:27,743 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/148e5a59aa0642c489ea33cad4750975 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/148e5a59aa0642c489ea33cad4750975 2024-12-04T15:00:27,744 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/273be2c0db2f478590487f934a9d149f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/273be2c0db2f478590487f934a9d149f 2024-12-04T15:00:27,745 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/8be9cb19ef804fd2a39ce2434421a8f7 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/8be9cb19ef804fd2a39ce2434421a8f7 2024-12-04T15:00:27,746 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/4250f3f1782a4156a421321bd4e88eb3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/4250f3f1782a4156a421321bd4e88eb3 2024-12-04T15:00:27,749 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/7f2a660307fa40759b0433e847fb89ad to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/7f2a660307fa40759b0433e847fb89ad 2024-12-04T15:00:27,750 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/1f33b79fb0ca40c1ae7aeb6e08661845 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/1f33b79fb0ca40c1ae7aeb6e08661845 2024-12-04T15:00:27,752 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/b6ecf3f70d51460db498888a1b6c9ee4 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/b6ecf3f70d51460db498888a1b6c9ee4 2024-12-04T15:00:27,754 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/50d2261ba36f46ca889107d330633ef8 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/50d2261ba36f46ca889107d330633ef8 2024-12-04T15:00:27,758 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/de7409bd89934bab813cce740a8dbc61 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/de7409bd89934bab813cce740a8dbc61 2024-12-04T15:00:27,765 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/0580fb20daae44a5988531b5a07a16ff to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/0580fb20daae44a5988531b5a07a16ff 2024-12-04T15:00:27,772 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/be3bdcd1f33f4a0fb4300e9ba7718566 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/be3bdcd1f33f4a0fb4300e9ba7718566 2024-12-04T15:00:27,779 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/a5449ad2ac564829a88ec8400ec3eee4 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/a5449ad2ac564829a88ec8400ec3eee4 2024-12-04T15:00:27,787 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/1af0459e7aad488ebe293fd0cdf2994e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/1af0459e7aad488ebe293fd0cdf2994e 2024-12-04T15:00:27,795 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/74804e4d808d430c86a62bb9d572d811 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/74804e4d808d430c86a62bb9d572d811 2024-12-04T15:00:27,807 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/fe538ebbda6b4d48bd024e8fcb21157c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/fe538ebbda6b4d48bd024e8fcb21157c 2024-12-04T15:00:27,813 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/6ed42f7bbfed4f978f19d7469efe74d8, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/51530c3cc9f744ddb614f31d5f58d07f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/644a962ccf5a4718a5dfbf5e454e7936, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/e67fd26e66ff4d96b5f1dcc2c3d66d3a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/20a264b6590b46338bb837d24f1cee53, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/b9a6f9490e034fc7a531fac8aa6f8166, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/3df8153fd5ea40e4a8922512058fbecd, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/e1abf86d7ca84a2aa55feeacc2b2c907, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/211f2de935224991a2b1088ad86da3ac, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/cd2db5594c084ad68e86983cfcef6809, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/efdb826eab0f46e896bd9cb98f4ce81b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/9c60eace0cd548fa9432c99e6bc37de2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/f47b4c76b44a480782f14a943ac9954e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/d23049b824134779a32abbc587eeb1f6, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/fa3f1a139c224072b013508a477b75a4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/0d0b219f56644a2ea6a16ab4083f0bb9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/f35dfcc461cb46a580aad9c31278cae4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/bf69a802ee1147d08fa0a82d11b88264, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/eaeb1a4606c040cea546d58f2cdc65c3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/481932a73a864f66939370875fd97fae, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/9539282decd04bd6b1c413a9dcbf1e78, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/efca068bc54343d6b87fab6bdda84fb3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/3fe2211dda284d3dbbaba5c68d6c1456, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/db49c109d6e64cbe934499367f75c445, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/5d250ea5eb5b482788d2ed166e38cda0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/0a2c3e1cabdc424bb5606b62a06a1342, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/c4cf383fff90482b9418eaa1f8a8a2de] to archive 2024-12-04T15:00:27,822 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T15:00:27,824 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/6ed42f7bbfed4f978f19d7469efe74d8 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/6ed42f7bbfed4f978f19d7469efe74d8 2024-12-04T15:00:27,825 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/51530c3cc9f744ddb614f31d5f58d07f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/51530c3cc9f744ddb614f31d5f58d07f 2024-12-04T15:00:27,826 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/644a962ccf5a4718a5dfbf5e454e7936 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/644a962ccf5a4718a5dfbf5e454e7936 2024-12-04T15:00:27,827 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/e67fd26e66ff4d96b5f1dcc2c3d66d3a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/e67fd26e66ff4d96b5f1dcc2c3d66d3a 2024-12-04T15:00:27,831 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/20a264b6590b46338bb837d24f1cee53 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/20a264b6590b46338bb837d24f1cee53 2024-12-04T15:00:27,833 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/b9a6f9490e034fc7a531fac8aa6f8166 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/b9a6f9490e034fc7a531fac8aa6f8166 2024-12-04T15:00:27,850 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/3df8153fd5ea40e4a8922512058fbecd to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/3df8153fd5ea40e4a8922512058fbecd 2024-12-04T15:00:27,863 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/e1abf86d7ca84a2aa55feeacc2b2c907 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/e1abf86d7ca84a2aa55feeacc2b2c907 2024-12-04T15:00:27,884 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/211f2de935224991a2b1088ad86da3ac to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/211f2de935224991a2b1088ad86da3ac 2024-12-04T15:00:27,896 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/cd2db5594c084ad68e86983cfcef6809 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/cd2db5594c084ad68e86983cfcef6809 2024-12-04T15:00:27,897 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/efdb826eab0f46e896bd9cb98f4ce81b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/efdb826eab0f46e896bd9cb98f4ce81b 2024-12-04T15:00:27,897 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/9c60eace0cd548fa9432c99e6bc37de2 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/9c60eace0cd548fa9432c99e6bc37de2 2024-12-04T15:00:27,901 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/f47b4c76b44a480782f14a943ac9954e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/f47b4c76b44a480782f14a943ac9954e 2024-12-04T15:00:27,902 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/d23049b824134779a32abbc587eeb1f6 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/d23049b824134779a32abbc587eeb1f6 2024-12-04T15:00:27,903 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/fa3f1a139c224072b013508a477b75a4 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/fa3f1a139c224072b013508a477b75a4 2024-12-04T15:00:27,904 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/0d0b219f56644a2ea6a16ab4083f0bb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/0d0b219f56644a2ea6a16ab4083f0bb9 2024-12-04T15:00:27,905 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/f35dfcc461cb46a580aad9c31278cae4 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/f35dfcc461cb46a580aad9c31278cae4 2024-12-04T15:00:27,906 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/bf69a802ee1147d08fa0a82d11b88264 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/bf69a802ee1147d08fa0a82d11b88264 2024-12-04T15:00:27,907 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/eaeb1a4606c040cea546d58f2cdc65c3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/eaeb1a4606c040cea546d58f2cdc65c3 2024-12-04T15:00:27,910 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/481932a73a864f66939370875fd97fae to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/481932a73a864f66939370875fd97fae 2024-12-04T15:00:27,912 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/9539282decd04bd6b1c413a9dcbf1e78 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/9539282decd04bd6b1c413a9dcbf1e78 2024-12-04T15:00:27,913 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/efca068bc54343d6b87fab6bdda84fb3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/efca068bc54343d6b87fab6bdda84fb3 2024-12-04T15:00:27,915 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/3fe2211dda284d3dbbaba5c68d6c1456 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/3fe2211dda284d3dbbaba5c68d6c1456 2024-12-04T15:00:27,916 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/db49c109d6e64cbe934499367f75c445 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/db49c109d6e64cbe934499367f75c445 2024-12-04T15:00:27,918 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/5d250ea5eb5b482788d2ed166e38cda0 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/5d250ea5eb5b482788d2ed166e38cda0 2024-12-04T15:00:27,984 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/0a2c3e1cabdc424bb5606b62a06a1342 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/0a2c3e1cabdc424bb5606b62a06a1342 2024-12-04T15:00:27,993 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/c4cf383fff90482b9418eaa1f8a8a2de to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/c4cf383fff90482b9418eaa1f8a8a2de 2024-12-04T15:00:33,305 INFO [Thread-1525 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1525. 2024-12-04T15:00:33,305 DEBUG [Thread-1525 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:00:33,305 DEBUG [Thread-1525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:33,305 DEBUG [Thread-1525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:33,306 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:00:33,334 INFO [Thread-1527 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1527. 2024-12-04T15:00:33,334 DEBUG [Thread-1527 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:00:33,334 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:33,334 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:33,334 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:00:33,374 INFO [Thread-1523 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1523. 2024-12-04T15:00:33,375 DEBUG [Thread-1523 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:00:33,375 DEBUG [Thread-1523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:33,375 DEBUG [Thread-1523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:33,375 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:00:33,393 INFO [Thread-1519 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1519. 2024-12-04T15:00:33,393 DEBUG [Thread-1519 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:00:33,393 DEBUG [Thread-1519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:33,394 DEBUG [Thread-1519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:33,394 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:00:33,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T15:00:33,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-04T15:00:33,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-12-04T15:00:33,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-12-04T15:00:33,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-12-04T15:00:33,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-12-04T15:00:33,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T15:00:33,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T15:00:33,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1298 2024-12-04T15:00:33,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3894 rows 2024-12-04T15:00:33,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1297 2024-12-04T15:00:33,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3891 rows 2024-12-04T15:00:33,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1290 2024-12-04T15:00:33,395 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3870 rows 2024-12-04T15:00:33,395 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1302 2024-12-04T15:00:33,395 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3906 rows 2024-12-04T15:00:33,395 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1291 2024-12-04T15:00:33,395 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3873 rows 2024-12-04T15:00:33,395 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T15:00:33,395 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T15:00:33,395 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:33,395 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:33,395 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:00:33,397 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T15:00:33,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T15:00:33,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T15:00:33,402 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324433402"}]},"ts":"1733324433402"} 2024-12-04T15:00:33,404 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T15:00:33,404 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T15:00:33,405 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T15:00:33,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e03f7dfff5f07d53203dd271e7128a0a, UNASSIGN}] 2024-12-04T15:00:33,408 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e03f7dfff5f07d53203dd271e7128a0a, UNASSIGN 2024-12-04T15:00:33,409 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=e03f7dfff5f07d53203dd271e7128a0a, regionState=CLOSING, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:33,411 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39449 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=d214edae7bd2,36649,1733324304801, table=TestAcidGuarantees, region=e03f7dfff5f07d53203dd271e7128a0a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-04T15:00:33,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e03f7dfff5f07d53203dd271e7128a0a, UNASSIGN because future has completed 2024-12-04T15:00:33,411 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T15:00:33,411 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801}] 2024-12-04T15:00:33,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T15:00:33,565 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(122): Close e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:33,565 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T15:00:33,565 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1722): Closing e03f7dfff5f07d53203dd271e7128a0a, disabling compactions & flushes 2024-12-04T15:00:33,566 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:33,566 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:33,566 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. after waiting 0 ms 2024-12-04T15:00:33,566 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:33,566 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(2902): Flushing e03f7dfff5f07d53203dd271e7128a0a 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-04T15:00:33,567 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=A 2024-12-04T15:00:33,567 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:33,567 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=B 2024-12-04T15:00:33,567 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:33,567 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e03f7dfff5f07d53203dd271e7128a0a, store=C 2024-12-04T15:00:33,567 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:33,576 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204149237efb1214349a96c4beb4cc0aca6_e03f7dfff5f07d53203dd271e7128a0a is 50, key is test_row_0/A:col10/1733324433302/Put/seqid=0 2024-12-04T15:00:33,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742273_1449 (size=12454) 2024-12-04T15:00:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T15:00:33,982 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:33,991 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204149237efb1214349a96c4beb4cc0aca6_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204149237efb1214349a96c4beb4cc0aca6_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:33,991 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/83e4736210264237b2cb4226f05899c1, store: [table=TestAcidGuarantees family=A region=e03f7dfff5f07d53203dd271e7128a0a] 2024-12-04T15:00:33,992 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/83e4736210264237b2cb4226f05899c1 is 175, key is test_row_0/A:col10/1733324433302/Put/seqid=0 2024-12-04T15:00:33,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742274_1450 (size=31255) 2024-12-04T15:00:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T15:00:34,396 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=433, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/83e4736210264237b2cb4226f05899c1 2024-12-04T15:00:34,412 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/c82d20dadc1d402281fe86c2afb50936 is 50, key is test_row_0/B:col10/1733324433302/Put/seqid=0 2024-12-04T15:00:34,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742275_1451 (size=12301) 2024-12-04T15:00:34,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T15:00:34,817 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/c82d20dadc1d402281fe86c2afb50936 2024-12-04T15:00:34,823 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/fd8682ccd37c46c4a4439633a67aa930 is 50, key is test_row_0/C:col10/1733324433302/Put/seqid=0 2024-12-04T15:00:34,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742276_1452 (size=12301) 2024-12-04T15:00:35,230 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/fd8682ccd37c46c4a4439633a67aa930 2024-12-04T15:00:35,238 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/A/83e4736210264237b2cb4226f05899c1 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/83e4736210264237b2cb4226f05899c1 2024-12-04T15:00:35,243 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/83e4736210264237b2cb4226f05899c1, entries=150, sequenceid=433, filesize=30.5 K 2024-12-04T15:00:35,244 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/B/c82d20dadc1d402281fe86c2afb50936 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c82d20dadc1d402281fe86c2afb50936 2024-12-04T15:00:35,248 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c82d20dadc1d402281fe86c2afb50936, entries=150, sequenceid=433, filesize=12.0 K 2024-12-04T15:00:35,249 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/.tmp/C/fd8682ccd37c46c4a4439633a67aa930 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/fd8682ccd37c46c4a4439633a67aa930 2024-12-04T15:00:35,253 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/fd8682ccd37c46c4a4439633a67aa930, entries=150, sequenceid=433, filesize=12.0 K 2024-12-04T15:00:35,254 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e03f7dfff5f07d53203dd271e7128a0a in 1688ms, sequenceid=433, compaction requested=true 2024-12-04T15:00:35,263 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/recovered.edits/436.seqid, newMaxSeqId=436, maxSeqId=4 2024-12-04T15:00:35,264 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. 2024-12-04T15:00:35,265 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1676): Region close journal for e03f7dfff5f07d53203dd271e7128a0a: Waiting for close lock at 1733324433565Running coprocessor pre-close hooks at 1733324433565Disabling compacts and flushes for region at 1733324433565Disabling writes for close at 1733324433566 (+1 ms)Obtaining lock to block concurrent updates at 1733324433566Preparing flush snapshotting stores in e03f7dfff5f07d53203dd271e7128a0a at 1733324433566Finished memstore snapshotting TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733324433567 (+1 ms)Flushing stores of TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a. at 1733324433569 (+2 ms)Flushing e03f7dfff5f07d53203dd271e7128a0a/A: creating writer at 1733324433569Flushing mob file e03f7dfff5f07d53203dd271e7128a0a/A: appending metadata at 1733324433576 (+7 ms)Flushing mob file e03f7dfff5f07d53203dd271e7128a0a/A: closing flushed file at 1733324433576Flushing e03f7dfff5f07d53203dd271e7128a0a/A: appending metadata at 1733324433991 (+415 ms)Flushing e03f7dfff5f07d53203dd271e7128a0a/A: closing flushed file at 1733324433991Flushing e03f7dfff5f07d53203dd271e7128a0a/B: creating writer at 1733324434405 (+414 ms)Flushing e03f7dfff5f07d53203dd271e7128a0a/B: appending metadata at 1733324434411 (+6 ms)Flushing e03f7dfff5f07d53203dd271e7128a0a/B: closing flushed file at 1733324434411Flushing e03f7dfff5f07d53203dd271e7128a0a/C: creating writer at 1733324434821 (+410 ms)Flushing e03f7dfff5f07d53203dd271e7128a0a/C: appending metadata at 1733324434823 (+2 ms)Flushing e03f7dfff5f07d53203dd271e7128a0a/C: closing flushed file at 1733324434823Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@532e3232: reopening flushed file at 1733324435238 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71e6089b: reopening flushed file at 1733324435243 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b0db0e9: reopening flushed file at 1733324435248 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e03f7dfff5f07d53203dd271e7128a0a in 1688ms, sequenceid=433, compaction requested=true at 1733324435254 (+6 ms)Writing region close event to WAL at 1733324435256 (+2 ms)Running coprocessor post-close hooks at 1733324435264 (+8 ms)Closed at 1733324435264 2024-12-04T15:00:35,267 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(157): Closed e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,267 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=e03f7dfff5f07d53203dd271e7128a0a, regionState=CLOSED 2024-12-04T15:00:35,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 because future has completed 2024-12-04T15:00:35,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-04T15:00:35,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseRegionProcedure e03f7dfff5f07d53203dd271e7128a0a, server=d214edae7bd2,36649,1733324304801 in 1.8640 sec 2024-12-04T15:00:35,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-12-04T15:00:35,281 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e03f7dfff5f07d53203dd271e7128a0a, UNASSIGN in 1.8710 sec 2024-12-04T15:00:35,285 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-04T15:00:35,285 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8770 sec 2024-12-04T15:00:35,286 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324435286"}]},"ts":"1733324435286"} 2024-12-04T15:00:35,288 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T15:00:35,288 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T15:00:35,289 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8910 sec 2024-12-04T15:00:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T15:00:35,551 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:35,552 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T15:00:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:00:35,554 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:00:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-04T15:00:35,555 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=109, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:00:35,557 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,558 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A, FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B, FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C, FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/recovered.edits] 2024-12-04T15:00:35,561 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/83e4736210264237b2cb4226f05899c1 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/83e4736210264237b2cb4226f05899c1 2024-12-04T15:00:35,562 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/ac26cdcf942346c58517a40452602c6a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/ac26cdcf942346c58517a40452602c6a 2024-12-04T15:00:35,563 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/b37c0d2473d3433daee5f4ab2fcb397a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/A/b37c0d2473d3433daee5f4ab2fcb397a 2024-12-04T15:00:35,566 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/633755811af14da69550b02518d820a3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/633755811af14da69550b02518d820a3 2024-12-04T15:00:35,567 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/aafefc8816004690851b9f2ee06e70a7 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/aafefc8816004690851b9f2ee06e70a7 2024-12-04T15:00:35,568 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c82d20dadc1d402281fe86c2afb50936 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/B/c82d20dadc1d402281fe86c2afb50936 2024-12-04T15:00:35,569 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/cd59eedc936042dca4699fc9e0c4563e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/cd59eedc936042dca4699fc9e0c4563e 2024-12-04T15:00:35,570 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/f5c4de57c47b430aac24d950a7248add to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/f5c4de57c47b430aac24d950a7248add 2024-12-04T15:00:35,571 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/fd8682ccd37c46c4a4439633a67aa930 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/C/fd8682ccd37c46c4a4439633a67aa930 2024-12-04T15:00:35,572 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/recovered.edits/436.seqid to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a/recovered.edits/436.seqid 2024-12-04T15:00:35,573 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,573 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T15:00:35,574 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T15:00:35,577 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-04T15:00:35,580 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040272d2e5d7f04c19a42144758a5bd12b_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040272d2e5d7f04c19a42144758a5bd12b_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,582 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040f80cb3dd5e144fc92934c01d5fdf414_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040f80cb3dd5e144fc92934c01d5fdf414_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,583 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041304fe8fb3cf4663840033cc2dc5a3be_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041304fe8fb3cf4663840033cc2dc5a3be_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,589 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204149237efb1214349a96c4beb4cc0aca6_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204149237efb1214349a96c4beb4cc0aca6_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,590 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041783a8301f02463ba6011c9c5392f98c_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041783a8301f02463ba6011c9c5392f98c_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,590 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042244aec9df674db5a2e9cb417be21f41_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042244aec9df674db5a2e9cb417be21f41_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,591 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412043030f3b1fcac48d9bd9a3e0c989fe496_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412043030f3b1fcac48d9bd9a3e0c989fe496_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,592 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120441537a463ee84bc194c39b5f0f34aea0_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120441537a463ee84bc194c39b5f0f34aea0_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,593 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204536d63ee54e54ce2afb5a66c90a9a02c_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204536d63ee54e54ce2afb5a66c90a9a02c_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,594 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045e0f1b9fdd264dee97cdf895ea2510aa_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045e0f1b9fdd264dee97cdf895ea2510aa_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,595 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046886ea7ff5914f4783d72d1978b59158_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046886ea7ff5914f4783d72d1978b59158_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,595 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047d3c1259afbb44a897a37e22b7a7f057_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047d3c1259afbb44a897a37e22b7a7f057_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,596 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204827386d5c506429888b935d140a185e7_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204827386d5c506429888b935d140a185e7_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,597 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204902fdc91b5624969b8bef9d345b810f1_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204902fdc91b5624969b8bef9d345b810f1_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,598 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a074286ae126413f804c9987a3460515_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a074286ae126413f804c9987a3460515_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,599 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a363828b52d74cf1841bcb9f04d0b7ec_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a363828b52d74cf1841bcb9f04d0b7ec_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,600 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204aaa1e5925fbc4059ba216032b5b904f7_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204aaa1e5925fbc4059ba216032b5b904f7_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,601 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204aef6328212bc4f369c50a99c90356234_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204aef6328212bc4f369c50a99c90356234_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,601 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c8476ae7a4064ae489308745fca2f98b_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c8476ae7a4064ae489308745fca2f98b_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,602 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204da7b4651c0594b2abedd25886bdcb8dd_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204da7b4651c0594b2abedd25886bdcb8dd_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,603 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e1c27340e10e4fa9b85a0765806db51f_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e1c27340e10e4fa9b85a0765806db51f_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,604 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204eb52c648df8d4aa28892b5a64465b0f4_e03f7dfff5f07d53203dd271e7128a0a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204eb52c648df8d4aa28892b5a64465b0f4_e03f7dfff5f07d53203dd271e7128a0a 2024-12-04T15:00:35,604 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T15:00:35,606 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=109, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:00:35,607 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T15:00:35,609 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T15:00:35,610 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=109, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:00:35,610 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T15:00:35,610 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733324435610"}]},"ts":"9223372036854775807"} 2024-12-04T15:00:35,612 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T15:00:35,612 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e03f7dfff5f07d53203dd271e7128a0a, NAME => 'TestAcidGuarantees,,1733324400350.e03f7dfff5f07d53203dd271e7128a0a.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T15:00:35,612 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T15:00:35,612 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733324435612"}]},"ts":"9223372036854775807"} 2024-12-04T15:00:35,614 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T15:00:35,614 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=109, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:00:35,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-12-04T15:00:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-04T15:00:35,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T15:00:35,661 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:35,670 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249 (was 250), OpenFileDescriptor=441 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=497 (was 532), ProcessCount=11 (was 11), AvailableMemoryMB=8454 (was 9795) 2024-12-04T15:00:35,677 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=497, ProcessCount=11, AvailableMemoryMB=8454 2024-12-04T15:00:35,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T15:00:35,678 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:00:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T15:00:35,679 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T15:00:35,679 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:35,679 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 110 2024-12-04T15:00:35,680 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T15:00:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-04T15:00:35,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742277_1453 (size=1061) 2024-12-04T15:00:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-04T15:00:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-04T15:00:36,087 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee 2024-12-04T15:00:36,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742278_1454 (size=53) 2024-12-04T15:00:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-04T15:00:36,492 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:00:36,492 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 15ba391c303ef265123c46f23b7fbad1, disabling compactions & flushes 2024-12-04T15:00:36,492 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:36,492 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:36,492 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. after waiting 0 ms 2024-12-04T15:00:36,492 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:36,492 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:36,493 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 15ba391c303ef265123c46f23b7fbad1: Waiting for close lock at 1733324436492Disabling compacts and flushes for region at 1733324436492Disabling writes for close at 1733324436492Writing region close event to WAL at 1733324436492Closed at 1733324436492 2024-12-04T15:00:36,494 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T15:00:36,494 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733324436494"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733324436494"}]},"ts":"1733324436494"} 2024-12-04T15:00:36,495 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T15:00:36,496 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T15:00:36,496 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324436496"}]},"ts":"1733324436496"} 2024-12-04T15:00:36,498 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T15:00:36,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=15ba391c303ef265123c46f23b7fbad1, ASSIGN}] 2024-12-04T15:00:36,501 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=15ba391c303ef265123c46f23b7fbad1, ASSIGN 2024-12-04T15:00:36,502 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=15ba391c303ef265123c46f23b7fbad1, ASSIGN; state=OFFLINE, location=d214edae7bd2,36649,1733324304801; forceNewPlan=false, retain=false 2024-12-04T15:00:36,653 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=15ba391c303ef265123c46f23b7fbad1, regionState=OPENING, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:36,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=15ba391c303ef265123c46f23b7fbad1, ASSIGN because future has completed 2024-12-04T15:00:36,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801}] 2024-12-04T15:00:36,813 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:36,814 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7752): Opening region: {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} 2024-12-04T15:00:36,815 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:36,815 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:00:36,816 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7794): checking encryption for 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:36,816 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7797): checking classloading for 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:36,818 INFO [StoreOpener-15ba391c303ef265123c46f23b7fbad1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:36,819 INFO [StoreOpener-15ba391c303ef265123c46f23b7fbad1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T15:00:36,819 INFO [StoreOpener-15ba391c303ef265123c46f23b7fbad1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15ba391c303ef265123c46f23b7fbad1 columnFamilyName A 2024-12-04T15:00:36,819 DEBUG [StoreOpener-15ba391c303ef265123c46f23b7fbad1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:36,820 INFO [StoreOpener-15ba391c303ef265123c46f23b7fbad1-1 {}] regionserver.HStore(327): Store=15ba391c303ef265123c46f23b7fbad1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:00:36,820 INFO [StoreOpener-15ba391c303ef265123c46f23b7fbad1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-04T15:00:36,821 INFO [StoreOpener-15ba391c303ef265123c46f23b7fbad1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T15:00:36,821 INFO [StoreOpener-15ba391c303ef265123c46f23b7fbad1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15ba391c303ef265123c46f23b7fbad1 columnFamilyName B 2024-12-04T15:00:36,822 DEBUG [StoreOpener-15ba391c303ef265123c46f23b7fbad1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:36,822 INFO [StoreOpener-15ba391c303ef265123c46f23b7fbad1-1 {}] regionserver.HStore(327): Store=15ba391c303ef265123c46f23b7fbad1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:00:36,822 INFO [StoreOpener-15ba391c303ef265123c46f23b7fbad1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:36,823 INFO [StoreOpener-15ba391c303ef265123c46f23b7fbad1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T15:00:36,823 INFO [StoreOpener-15ba391c303ef265123c46f23b7fbad1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15ba391c303ef265123c46f23b7fbad1 columnFamilyName C 2024-12-04T15:00:36,823 DEBUG [StoreOpener-15ba391c303ef265123c46f23b7fbad1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:00:36,824 INFO [StoreOpener-15ba391c303ef265123c46f23b7fbad1-1 {}] regionserver.HStore(327): Store=15ba391c303ef265123c46f23b7fbad1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:00:36,824 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:36,824 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1038): replaying wal for 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:36,825 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:36,826 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:36,827 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1048): stopping wal replay for 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:36,827 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1060): Cleaning up temporary data for 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:36,827 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T15:00:36,828 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1093): writing seq id for 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:36,830 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T15:00:36,830 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1114): Opened 15ba391c303ef265123c46f23b7fbad1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66033899, jitterRate=-0.016018226742744446}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T15:00:36,830 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:36,831 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1006): Region open journal for 15ba391c303ef265123c46f23b7fbad1: Running coprocessor pre-open hook at 1733324436816Writing region info on filesystem at 1733324436816Initializing all the Stores at 1733324436817 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324436817Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324436818 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324436818Cleaning up temporary data from old regions at 1733324436827 (+9 ms)Running coprocessor post-open hooks at 1733324436830 (+3 ms)Region opened successfully at 1733324436831 (+1 ms) 2024-12-04T15:00:36,832 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., pid=112, masterSystemTime=1733324436809 2024-12-04T15:00:36,834 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:36,834 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:36,834 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=15ba391c303ef265123c46f23b7fbad1, regionState=OPEN, openSeqNum=2, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:36,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 because future has completed 2024-12-04T15:00:36,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-04T15:00:36,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; OpenRegionProcedure 15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 in 181 msec 2024-12-04T15:00:36,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-12-04T15:00:36,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=15ba391c303ef265123c46f23b7fbad1, ASSIGN in 339 msec 2024-12-04T15:00:36,841 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T15:00:36,841 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324436841"}]},"ts":"1733324436841"} 2024-12-04T15:00:36,842 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T15:00:36,843 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T15:00:36,845 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-12-04T15:00:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-04T15:00:37,832 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:37,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ca175cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,836 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:37,836 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:37,839 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:37,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:37,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@714ff7a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:37,842 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54498, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:37,842 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@570c3fc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,842 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:37,844 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:37,845 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T15:00:37,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is d214edae7bd2,39449,1733324304115 2024-12-04T15:00:37,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@637cb582 2024-12-04T15:00:37,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T15:00:37,846 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T15:00:37,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@491481f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,848 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:37,848 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:37,848 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ebc6a66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:37,850 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29efb1e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7631696d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,851 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:37,851 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:37,852 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12ea41d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:37,853 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36878, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@914aa9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,854 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@126b8236, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,854 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:37,854 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:37,854 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b72d89a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:37,856 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36902, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b2ee80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,857 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ea00808, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,857 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:37,857 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:37,857 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e045e17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:37,859 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:37,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@247eefed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1589e05f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,859 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:37,860 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:37,860 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@acaad05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:37,861 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:37,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e3adc4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44c19380, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,862 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:37,863 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:37,863 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:37,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d24e755, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:37,865 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7463ca73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b23f2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,866 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:37,866 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:37,867 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:37,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:37,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:37,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@269bb9ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:37,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:37,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:37,868 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:37,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d74ba31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c7d791a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,870 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:37,870 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:37,870 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:37,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:37,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:37,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4092745c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:37,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:37,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:37,872 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:37,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d4126f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57b3a94b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,873 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:37,873 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:37,874 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:37,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:37,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:37,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fee49f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:37,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:37,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:37,875 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:37,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b2de086, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,877 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c68a1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,877 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:00:37,877 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:00:37,878 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:00:37,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:00:37,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:00:37,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6764c769, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:00:37,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:00:37,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:37,879 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:00:37,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d3d9d9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:00:37,881 DEBUG [Thread-2040 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:37,881 DEBUG [Thread-2042 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:37,881 DEBUG [Thread-2051 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:37,881 DEBUG [Thread-2049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:37,881 DEBUG [Thread-2044 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:37,882 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:00:37,882 DEBUG [Thread-2047 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:37,883 DEBUG [Thread-2053 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:37,883 DEBUG [Thread-2055 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:37,883 DEBUG [Thread-2036 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-12-04T15:00:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:37,884 DEBUG [Thread-2053 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:37,885 DEBUG [Thread-2053 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:37,885 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:37,885 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:37,885 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-04T15:00:37,885 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:37,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:37,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:37,886 DEBUG [Thread-2038 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:00:37,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:37,886 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:00:37,887 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:37,887 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:00:37,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T15:00:37,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:00:37,887 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:37,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T15:00:37,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T15:00:37,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:00:37,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:00:37,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T15:00:37,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T15:00:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T15:00:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T15:00:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T15:00:37,891 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:00:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T15:00:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2] 2024-12-04T15:00:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:37,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T15:00:37,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:37,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:37,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:37,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:37,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:37,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:37,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/a5946126bb9849e099defd46ec47f273 is 50, key is test_row_0/A:col10/1733324437898/Put/seqid=0 2024-12-04T15:00:37,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742279_1455 (size=12001) 2024-12-04T15:00:37,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/a5946126bb9849e099defd46ec47f273 2024-12-04T15:00:37,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324497929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:37,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733324437923, max timestamp=1733324437936Num uniques 859; 2024-12-04T15:00:37,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T15:00:37,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733324437923, max timestamp=1733324437936Num uniques 859; 2024-12-04T15:00:37,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T15:00:37,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733324437923, max timestamp=1733324437936Num uniques 859; 2024-12-04T15:00:37,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T15:00:37,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:37,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:37,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:37,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/e01a0b55da9845318cb2b4c637698ac0 is 50, key is test_row_0/B:col10/1733324437898/Put/seqid=0 2024-12-04T15:00:37,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742280_1456 (size=12001) 2024-12-04T15:00:37,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324497966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:37,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:37,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1733324497967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:37,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:37,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:37,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1733324497970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:37,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:37,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:37,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:37,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:37,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:37,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:37,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45062 deadline: 1733324497971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:37,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:37,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:37,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-04T15:00:38,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-04T15:00:38,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:38,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:38,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:38,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:38,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:38,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324498049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:38,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324498080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:38,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:38,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1733324498080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:38,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:38,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1733324498080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:38,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:38,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45062 deadline: 1733324498080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:38,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:38,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-04T15:00:38,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:38,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:38,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:38,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:38,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:38,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:38,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-04T15:00:38,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:38,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324498260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:38,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:38,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324498290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:38,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1733324498290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1733324498290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:38,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45062 deadline: 1733324498290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:38,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-04T15:00:38,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:38,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:38,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:38,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:38,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:38,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:38,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/e01a0b55da9845318cb2b4c637698ac0 2024-12-04T15:00:38,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/379a7d2565f547adbda831707800be61 is 50, key is test_row_0/C:col10/1733324437898/Put/seqid=0 2024-12-04T15:00:38,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742281_1457 (size=12001) 2024-12-04T15:00:38,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/379a7d2565f547adbda831707800be61 2024-12-04T15:00:38,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/a5946126bb9849e099defd46ec47f273 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/a5946126bb9849e099defd46ec47f273 2024-12-04T15:00:38,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/a5946126bb9849e099defd46ec47f273, entries=150, sequenceid=12, filesize=11.7 K 2024-12-04T15:00:38,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/e01a0b55da9845318cb2b4c637698ac0 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/e01a0b55da9845318cb2b4c637698ac0 2024-12-04T15:00:38,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/e01a0b55da9845318cb2b4c637698ac0, entries=150, sequenceid=12, filesize=11.7 K 2024-12-04T15:00:38,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/379a7d2565f547adbda831707800be61 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/379a7d2565f547adbda831707800be61 2024-12-04T15:00:38,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/379a7d2565f547adbda831707800be61, entries=150, sequenceid=12, filesize=11.7 K 2024-12-04T15:00:38,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 15ba391c303ef265123c46f23b7fbad1 in 515ms, sequenceid=12, compaction requested=false 2024-12-04T15:00:38,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:38,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-04T15:00:38,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:38,500 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-04T15:00:38,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:38,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:38,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:38,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:38,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:38,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:38,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/e215e39554eb4fafbbd78428b290eb47 is 50, key is test_row_0/A:col10/1733324437967/Put/seqid=0 2024-12-04T15:00:38,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742282_1458 (size=12001) 2024-12-04T15:00:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-04T15:00:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:38,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:38,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1733324498600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1733324498600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:38,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:38,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45062 deadline: 1733324498600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:38,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:38,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324498600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:38,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:38,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324498605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:38,614 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:38,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324498710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:38,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324498720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:38,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:38,908 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/e215e39554eb4fafbbd78428b290eb47 2024-12-04T15:00:38,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/c379ce69dce24048b91cb2bb91e89e7f is 50, key is test_row_0/B:col10/1733324437967/Put/seqid=0 2024-12-04T15:00:38,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742283_1459 (size=12001) 2024-12-04T15:00:38,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:38,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324498930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:38,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:38,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:38,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324498930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:38,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:38,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:39,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-04T15:00:39,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45062 deadline: 1733324499111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:39,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1733324499120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:39,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1733324499120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:39,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:39,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:39,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:39,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324499240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:39,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324499240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:39,317 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/c379ce69dce24048b91cb2bb91e89e7f 2024-12-04T15:00:39,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/592e1268a1e14bdc8e0fa24767059214 is 50, key is test_row_0/C:col10/1733324437967/Put/seqid=0 2024-12-04T15:00:39,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742284_1460 (size=12001) 2024-12-04T15:00:39,725 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/592e1268a1e14bdc8e0fa24767059214 2024-12-04T15:00:39,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/e215e39554eb4fafbbd78428b290eb47 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/e215e39554eb4fafbbd78428b290eb47 2024-12-04T15:00:39,732 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/e215e39554eb4fafbbd78428b290eb47, entries=150, sequenceid=46, filesize=11.7 K 2024-12-04T15:00:39,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/c379ce69dce24048b91cb2bb91e89e7f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/c379ce69dce24048b91cb2bb91e89e7f 2024-12-04T15:00:39,735 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/c379ce69dce24048b91cb2bb91e89e7f, entries=150, sequenceid=46, filesize=11.7 K 2024-12-04T15:00:39,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/592e1268a1e14bdc8e0fa24767059214 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/592e1268a1e14bdc8e0fa24767059214 2024-12-04T15:00:39,738 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/592e1268a1e14bdc8e0fa24767059214, entries=150, sequenceid=46, filesize=11.7 K 2024-12-04T15:00:39,739 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 15ba391c303ef265123c46f23b7fbad1 in 1239ms, sequenceid=46, compaction requested=false 2024-12-04T15:00:39,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:39,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:39,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-12-04T15:00:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-12-04T15:00:39,742 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-04T15:00:39,742 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8530 sec 2024-12-04T15:00:39,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.8610 sec 2024-12-04T15:00:39,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:39,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T15:00:39,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:39,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:39,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:39,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:39,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:39,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:39,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/9588e4d3d81c42048b75fd7ec2c9d3af is 50, key is test_row_0/A:col10/1733324439751/Put/seqid=0 2024-12-04T15:00:39,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742285_1461 (size=14341) 2024-12-04T15:00:39,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324499810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:39,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:39,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:39,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:39,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324499818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:39,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:39,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:39,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:39,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324499920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:39,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:39,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:39,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:39,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324499930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-04T15:00:40,031 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:40,032 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:00:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-04T15:00:40,035 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:00:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T15:00:40,035 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:00:40,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:00:40,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45062 deadline: 1733324500120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:40,132 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 15ba391c303ef265123c46f23b7fbad1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:40,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:40,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1733324500130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:40,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324500131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:40,133 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_0' in 15ba391c303ef265123c46f23b7fbad1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:40,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:40,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T15:00:40,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1733324500140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:40,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324500140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:40,148 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_0' in 15ba391c303ef265123c46f23b7fbad1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:40,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:40,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:40,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/9588e4d3d81c42048b75fd7ec2c9d3af 2024-12-04T15:00:40,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/20aec5b168ac4915a3104204c9893fb5 is 50, key is test_row_0/B:col10/1733324439751/Put/seqid=0 2024-12-04T15:00:40,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T15:00:40,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:40,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:40,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:40,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:40,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:40,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742286_1462 (size=12001) 2024-12-04T15:00:40,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/20aec5b168ac4915a3104204c9893fb5 2024-12-04T15:00:40,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/75fcff6ff1e34560ad835e89d4a9b031 is 50, key is test_row_0/C:col10/1733324439751/Put/seqid=0 2024-12-04T15:00:40,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742287_1463 (size=12001) 2024-12-04T15:00:40,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/75fcff6ff1e34560ad835e89d4a9b031 2024-12-04T15:00:40,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/9588e4d3d81c42048b75fd7ec2c9d3af as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/9588e4d3d81c42048b75fd7ec2c9d3af 2024-12-04T15:00:40,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/9588e4d3d81c42048b75fd7ec2c9d3af, entries=200, sequenceid=58, filesize=14.0 K 2024-12-04T15:00:40,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/20aec5b168ac4915a3104204c9893fb5 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/20aec5b168ac4915a3104204c9893fb5 2024-12-04T15:00:40,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/20aec5b168ac4915a3104204c9893fb5, entries=150, sequenceid=58, filesize=11.7 K 2024-12-04T15:00:40,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/75fcff6ff1e34560ad835e89d4a9b031 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/75fcff6ff1e34560ad835e89d4a9b031 2024-12-04T15:00:40,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/75fcff6ff1e34560ad835e89d4a9b031, entries=150, sequenceid=58, filesize=11.7 K 2024-12-04T15:00:40,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 15ba391c303ef265123c46f23b7fbad1 in 542ms, sequenceid=58, compaction requested=true 2024-12-04T15:00:40,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:40,297 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:40,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:00:40,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:40,297 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:40,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:00:40,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:40,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:00:40,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:40,298 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:40,298 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/A is initiating minor compaction (all files) 2024-12-04T15:00:40,298 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/A in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:40,298 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:40,298 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/a5946126bb9849e099defd46ec47f273, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/e215e39554eb4fafbbd78428b290eb47, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/9588e4d3d81c42048b75fd7ec2c9d3af] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=37.4 K 2024-12-04T15:00:40,298 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/B is initiating minor compaction (all files) 2024-12-04T15:00:40,298 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/B in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:40,298 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/e01a0b55da9845318cb2b4c637698ac0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/c379ce69dce24048b91cb2bb91e89e7f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/20aec5b168ac4915a3104204c9893fb5] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=35.2 K 2024-12-04T15:00:40,299 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5946126bb9849e099defd46ec47f273, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733324437893 2024-12-04T15:00:40,299 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting e01a0b55da9845318cb2b4c637698ac0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733324437893 2024-12-04T15:00:40,299 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e215e39554eb4fafbbd78428b290eb47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733324437965 2024-12-04T15:00:40,299 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting c379ce69dce24048b91cb2bb91e89e7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733324437965 2024-12-04T15:00:40,299 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 20aec5b168ac4915a3104204c9893fb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733324438577 2024-12-04T15:00:40,299 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9588e4d3d81c42048b75fd7ec2c9d3af, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733324438577 2024-12-04T15:00:40,334 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#B#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:40,335 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/cc80b514be0445279efe426a0d334bca is 50, key is test_row_0/B:col10/1733324439751/Put/seqid=0 2024-12-04T15:00:40,349 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#A#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:40,350 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/be8b4f37a4c645bc97a55791301047c0 is 50, key is test_row_0/A:col10/1733324439751/Put/seqid=0 2024-12-04T15:00:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T15:00:40,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T15:00:40,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:40,351 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T15:00:40,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:40,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:40,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:40,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:40,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:40,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:40,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/6b4769bf52304c8792191940a67b37bc is 50, key is test_row_0/A:col10/1733324439800/Put/seqid=0 2024-12-04T15:00:40,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742289_1465 (size=12104) 2024-12-04T15:00:40,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742288_1464 (size=12104) 2024-12-04T15:00:40,453 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/cc80b514be0445279efe426a0d334bca as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/cc80b514be0445279efe426a0d334bca 2024-12-04T15:00:40,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:40,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:40,458 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/B of 15ba391c303ef265123c46f23b7fbad1 into cc80b514be0445279efe426a0d334bca(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:40,458 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:40,458 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/B, priority=13, startTime=1733324440297; duration=0sec 2024-12-04T15:00:40,458 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:40,459 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:B 2024-12-04T15:00:40,459 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:40,460 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:40,460 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/C is initiating minor compaction (all files) 2024-12-04T15:00:40,460 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/C in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:40,460 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/379a7d2565f547adbda831707800be61, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/592e1268a1e14bdc8e0fa24767059214, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/75fcff6ff1e34560ad835e89d4a9b031] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=35.2 K 2024-12-04T15:00:40,460 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 379a7d2565f547adbda831707800be61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733324437893 2024-12-04T15:00:40,461 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 592e1268a1e14bdc8e0fa24767059214, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733324437965 2024-12-04T15:00:40,461 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 75fcff6ff1e34560ad835e89d4a9b031, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733324438577 2024-12-04T15:00:40,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742290_1466 (size=12001) 2024-12-04T15:00:40,501 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#C#compaction#387 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:40,501 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/6947e91cb1a6449c9caf6d7787a3fabd is 50, key is test_row_0/C:col10/1733324439751/Put/seqid=0 2024-12-04T15:00:40,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742291_1467 (size=12104) 2024-12-04T15:00:40,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:40,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324500562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:40,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:40,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:40,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324500567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:40,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:40,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T15:00:40,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324500690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:40,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:40,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324500710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:40,799 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T15:00:40,850 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/be8b4f37a4c645bc97a55791301047c0 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/be8b4f37a4c645bc97a55791301047c0 2024-12-04T15:00:40,857 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/A of 15ba391c303ef265123c46f23b7fbad1 into be8b4f37a4c645bc97a55791301047c0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:40,857 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:40,857 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/A, priority=13, startTime=1733324440297; duration=0sec 2024-12-04T15:00:40,857 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:40,857 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:A 2024-12-04T15:00:40,870 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/6b4769bf52304c8792191940a67b37bc 2024-12-04T15:00:40,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/dee450952f824f41a1b680dd63b4c7f4 is 50, key is test_row_0/B:col10/1733324439800/Put/seqid=0 2024-12-04T15:00:40,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742292_1468 (size=12001) 2024-12-04T15:00:40,922 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/dee450952f824f41a1b680dd63b4c7f4 2024-12-04T15:00:40,972 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/6947e91cb1a6449c9caf6d7787a3fabd as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/6947e91cb1a6449c9caf6d7787a3fabd 2024-12-04T15:00:40,976 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/C of 15ba391c303ef265123c46f23b7fbad1 into 6947e91cb1a6449c9caf6d7787a3fabd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:40,976 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:40,976 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/C, priority=13, startTime=1733324440297; duration=0sec 2024-12-04T15:00:40,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/05730b1c73df415cae157fffebb34623 is 50, key is test_row_0/C:col10/1733324439800/Put/seqid=0 2024-12-04T15:00:40,976 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:40,976 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:C 2024-12-04T15:00:40,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324500930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:40,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324500930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:40,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:40,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:40,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:41,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742293_1469 (size=12001) 2024-12-04T15:00:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T15:00:41,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324501281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:41,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324501291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:41,424 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/05730b1c73df415cae157fffebb34623 2024-12-04T15:00:41,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/6b4769bf52304c8792191940a67b37bc as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/6b4769bf52304c8792191940a67b37bc 2024-12-04T15:00:41,431 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/6b4769bf52304c8792191940a67b37bc, entries=150, sequenceid=78, filesize=11.7 K 2024-12-04T15:00:41,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/dee450952f824f41a1b680dd63b4c7f4 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/dee450952f824f41a1b680dd63b4c7f4 2024-12-04T15:00:41,435 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/dee450952f824f41a1b680dd63b4c7f4, entries=150, sequenceid=78, filesize=11.7 K 2024-12-04T15:00:41,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/05730b1c73df415cae157fffebb34623 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/05730b1c73df415cae157fffebb34623 2024-12-04T15:00:41,438 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/05730b1c73df415cae157fffebb34623, entries=150, sequenceid=78, filesize=11.7 K 2024-12-04T15:00:41,439 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 15ba391c303ef265123c46f23b7fbad1 in 1088ms, sequenceid=78, compaction requested=false 2024-12-04T15:00:41,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:41,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:41,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-04T15:00:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-04T15:00:41,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-04T15:00:41,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4050 sec 2024-12-04T15:00:41,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.4100 sec 2024-12-04T15:00:41,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:41,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T15:00:41,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:41,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:41,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:41,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:41,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:41,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:41,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/55b6d3fb71544c4e99af027507487f63 is 50, key is test_row_0/A:col10/1733324441802/Put/seqid=0 2024-12-04T15:00:41,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742294_1470 (size=16681) 2024-12-04T15:00:41,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/55b6d3fb71544c4e99af027507487f63 2024-12-04T15:00:41,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/62e5a881889b41819dd907998c029f22 is 50, key is test_row_0/B:col10/1733324441802/Put/seqid=0 2024-12-04T15:00:41,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742295_1471 (size=12001) 2024-12-04T15:00:41,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/62e5a881889b41819dd907998c029f22 2024-12-04T15:00:41,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/c561f1d255f2419abe56b618fc68a7ff is 50, key is test_row_0/C:col10/1733324441802/Put/seqid=0 2024-12-04T15:00:41,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742296_1472 (size=12001) 2024-12-04T15:00:41,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/c561f1d255f2419abe56b618fc68a7ff 2024-12-04T15:00:41,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/55b6d3fb71544c4e99af027507487f63 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/55b6d3fb71544c4e99af027507487f63 2024-12-04T15:00:41,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/55b6d3fb71544c4e99af027507487f63, entries=250, sequenceid=95, filesize=16.3 K 2024-12-04T15:00:41,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/62e5a881889b41819dd907998c029f22 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/62e5a881889b41819dd907998c029f22 2024-12-04T15:00:41,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/62e5a881889b41819dd907998c029f22, entries=150, sequenceid=95, filesize=11.7 K 2024-12-04T15:00:41,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/c561f1d255f2419abe56b618fc68a7ff as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/c561f1d255f2419abe56b618fc68a7ff 2024-12-04T15:00:41,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/c561f1d255f2419abe56b618fc68a7ff, entries=150, sequenceid=95, filesize=11.7 K 2024-12-04T15:00:41,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 15ba391c303ef265123c46f23b7fbad1 in 62ms, sequenceid=95, compaction requested=true 2024-12-04T15:00:41,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:41,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:00:41,866 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:41,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:41,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:00:41,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:41,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:00:41,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:41,867 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:41,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-04T15:00:41,867 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:41,867 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:41,867 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/A is initiating minor compaction (all files) 2024-12-04T15:00:41,867 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/B is initiating minor compaction (all files) 2024-12-04T15:00:41,867 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/B in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:41,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:41,867 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/A in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:41,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:41,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:41,867 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/be8b4f37a4c645bc97a55791301047c0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/6b4769bf52304c8792191940a67b37bc, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/55b6d3fb71544c4e99af027507487f63] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=39.8 K 2024-12-04T15:00:41,867 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/cc80b514be0445279efe426a0d334bca, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/dee450952f824f41a1b680dd63b4c7f4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/62e5a881889b41819dd907998c029f22] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=35.3 K 2024-12-04T15:00:41,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:41,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:41,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:41,868 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting be8b4f37a4c645bc97a55791301047c0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733324438577 2024-12-04T15:00:41,868 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting cc80b514be0445279efe426a0d334bca, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733324438577 2024-12-04T15:00:41,868 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting dee450952f824f41a1b680dd63b4c7f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733324439800 2024-12-04T15:00:41,868 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b4769bf52304c8792191940a67b37bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733324439800 2024-12-04T15:00:41,868 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55b6d3fb71544c4e99af027507487f63, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733324440501 2024-12-04T15:00:41,868 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 62e5a881889b41819dd907998c029f22, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733324440535 2024-12-04T15:00:41,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/36391914a4f4458d840b0907732e6306 is 50, key is test_row_0/A:col10/1733324441852/Put/seqid=0 2024-12-04T15:00:41,874 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#A#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:41,875 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/b5f4fcc78a0347cc8d72367d4458db84 is 50, key is test_row_0/A:col10/1733324441802/Put/seqid=0 2024-12-04T15:00:41,876 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#B#compaction#395 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:41,877 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/4a03a3ea6abb41d8be8e86edc4ab8c43 is 50, key is test_row_0/B:col10/1733324441802/Put/seqid=0 2024-12-04T15:00:41,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742297_1473 (size=16681) 2024-12-04T15:00:41,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742298_1474 (size=12207) 2024-12-04T15:00:41,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/36391914a4f4458d840b0907732e6306 2024-12-04T15:00:41,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742299_1475 (size=12207) 2024-12-04T15:00:41,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/5afb594ccf5b4da4bac90b89dd37311c is 50, key is test_row_0/B:col10/1733324441852/Put/seqid=0 2024-12-04T15:00:41,892 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/4a03a3ea6abb41d8be8e86edc4ab8c43 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/4a03a3ea6abb41d8be8e86edc4ab8c43 2024-12-04T15:00:41,897 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/b5f4fcc78a0347cc8d72367d4458db84 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/b5f4fcc78a0347cc8d72367d4458db84 2024-12-04T15:00:41,899 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/B of 15ba391c303ef265123c46f23b7fbad1 into 4a03a3ea6abb41d8be8e86edc4ab8c43(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:41,899 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:41,899 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/B, priority=13, startTime=1733324441866; duration=0sec 2024-12-04T15:00:41,899 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:41,899 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:B 2024-12-04T15:00:41,899 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:41,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742300_1476 (size=12001) 2024-12-04T15:00:41,900 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:41,900 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/C is initiating minor compaction (all files) 2024-12-04T15:00:41,900 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/C in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:41,900 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/6947e91cb1a6449c9caf6d7787a3fabd, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/05730b1c73df415cae157fffebb34623, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/c561f1d255f2419abe56b618fc68a7ff] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=35.3 K 2024-12-04T15:00:41,901 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 6947e91cb1a6449c9caf6d7787a3fabd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733324438577 2024-12-04T15:00:41,901 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 05730b1c73df415cae157fffebb34623, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733324439800 2024-12-04T15:00:41,901 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting c561f1d255f2419abe56b618fc68a7ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733324440535 2024-12-04T15:00:41,904 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/A of 15ba391c303ef265123c46f23b7fbad1 into b5f4fcc78a0347cc8d72367d4458db84(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:41,904 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:41,904 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/A, priority=13, startTime=1733324441866; duration=0sec 2024-12-04T15:00:41,905 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:41,905 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:A 2024-12-04T15:00:41,907 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#C#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:41,908 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/e3c10235dce34771829ca0ffe7d175f6 is 50, key is test_row_0/C:col10/1733324441802/Put/seqid=0 2024-12-04T15:00:41,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742301_1477 (size=12207) 2024-12-04T15:00:41,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324501923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:41,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324501924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:41,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:41,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:41,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:42,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324502030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:42,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:42,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324502040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:42,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:42,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:42,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:42,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45062 deadline: 1733324502160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:42,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1733324502160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:42,161 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 15ba391c303ef265123c46f23b7fbad1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:42,161 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_0' in 15ba391c303ef265123c46f23b7fbad1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:42,162 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:42,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:42,162 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:42,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:42,162 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:42,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:42,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1733324502170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:42,175 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_0' in 15ba391c303ef265123c46f23b7fbad1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:42,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:42,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:42,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T15:00:42,181 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:42,181 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:00:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-04T15:00:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T15:00:42,183 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:00:42,183 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:00:42,183 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:00:42,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324502240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:42,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:42,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324502250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:42,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T15:00:42,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/5afb594ccf5b4da4bac90b89dd37311c 2024-12-04T15:00:42,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/acb6cf2c8949423bbf63df6e0245d276 is 50, key is test_row_0/C:col10/1733324441852/Put/seqid=0 2024-12-04T15:00:42,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742302_1478 (size=12001) 2024-12-04T15:00:42,313 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/e3c10235dce34771829ca0ffe7d175f6 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/e3c10235dce34771829ca0ffe7d175f6 2024-12-04T15:00:42,316 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/C of 15ba391c303ef265123c46f23b7fbad1 into e3c10235dce34771829ca0ffe7d175f6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:42,316 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:42,316 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/C, priority=13, startTime=1733324441867; duration=0sec 2024-12-04T15:00:42,316 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:42,316 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:C 2024-12-04T15:00:42,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-04T15:00:42,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:42,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:42,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:42,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:42,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:42,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-04T15:00:42,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:42,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:42,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:42,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:42,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T15:00:42,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324502549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:42,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:42,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:42,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:42,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324502560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:42,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-04T15:00:42,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:42,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:42,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:42,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:42,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:42,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/acb6cf2c8949423bbf63df6e0245d276 2024-12-04T15:00:42,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/36391914a4f4458d840b0907732e6306 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/36391914a4f4458d840b0907732e6306 2024-12-04T15:00:42,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/36391914a4f4458d840b0907732e6306, entries=250, sequenceid=114, filesize=16.3 K 2024-12-04T15:00:42,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/5afb594ccf5b4da4bac90b89dd37311c as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/5afb594ccf5b4da4bac90b89dd37311c 2024-12-04T15:00:42,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/5afb594ccf5b4da4bac90b89dd37311c, entries=150, sequenceid=114, filesize=11.7 K 2024-12-04T15:00:42,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/acb6cf2c8949423bbf63df6e0245d276 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/acb6cf2c8949423bbf63df6e0245d276 2024-12-04T15:00:42,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/acb6cf2c8949423bbf63df6e0245d276, entries=150, sequenceid=114, filesize=11.7 K 2024-12-04T15:00:42,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 15ba391c303ef265123c46f23b7fbad1 in 860ms, sequenceid=114, compaction requested=false 2024-12-04T15:00:42,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:42,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-04T15:00:42,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:42,794 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-04T15:00:42,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:42,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:42,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:42,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:42,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:42,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:42,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/da6ce47c7710483bb1732bcbc59cc188 is 50, key is test_row_0/A:col10/1733324441922/Put/seqid=0 2024-12-04T15:00:42,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T15:00:42,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742303_1479 (size=12051) 2024-12-04T15:00:43,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:43,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:43,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:43,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324503104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:43,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:43,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:43,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:43,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324503114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:43,214 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/da6ce47c7710483bb1732bcbc59cc188 2024-12-04T15:00:43,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/271c0bae28e04b958f57c4b872aee8ac is 50, key is test_row_0/B:col10/1733324441922/Put/seqid=0 2024-12-04T15:00:43,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742304_1480 (size=12051) 2024-12-04T15:00:43,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324503222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:43,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324503230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:43,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:43,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:43,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T15:00:43,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:43,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324503440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:43,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:43,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324503440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:43,624 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/271c0bae28e04b958f57c4b872aee8ac 2024-12-04T15:00:43,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/40727f1bae864819b1e2daed6ab241c2 is 50, key is test_row_0/C:col10/1733324441922/Put/seqid=0 2024-12-04T15:00:43,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742305_1481 (size=12051) 2024-12-04T15:00:43,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324503750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:43,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:43,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:43,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324503750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:43,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:43,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:43,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:43,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:44,033 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/40727f1bae864819b1e2daed6ab241c2 2024-12-04T15:00:44,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/da6ce47c7710483bb1732bcbc59cc188 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/da6ce47c7710483bb1732bcbc59cc188 2024-12-04T15:00:44,039 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/da6ce47c7710483bb1732bcbc59cc188, entries=150, sequenceid=131, filesize=11.8 K 2024-12-04T15:00:44,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/271c0bae28e04b958f57c4b872aee8ac as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/271c0bae28e04b958f57c4b872aee8ac 2024-12-04T15:00:44,042 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/271c0bae28e04b958f57c4b872aee8ac, entries=150, sequenceid=131, filesize=11.8 K 2024-12-04T15:00:44,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/40727f1bae864819b1e2daed6ab241c2 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/40727f1bae864819b1e2daed6ab241c2 2024-12-04T15:00:44,047 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/40727f1bae864819b1e2daed6ab241c2, entries=150, sequenceid=131, filesize=11.8 K 2024-12-04T15:00:44,047 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 15ba391c303ef265123c46f23b7fbad1 in 1253ms, sequenceid=131, compaction requested=true 2024-12-04T15:00:44,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:44,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:44,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-04T15:00:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-04T15:00:44,052 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-04T15:00:44,052 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8670 sec 2024-12-04T15:00:44,053 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.8710 sec 2024-12-04T15:00:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:44,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-04T15:00:44,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:44,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:44,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:44,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:44,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:44,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:44,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/ebfbf5f55905415ab0cf940dbd688a8f is 50, key is test_row_0/A:col10/1733324444273/Put/seqid=0 2024-12-04T15:00:44,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742306_1482 (size=14541) 2024-12-04T15:00:44,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324504309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:44,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:44,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:44,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:44,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324504311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:44,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T15:00:44,330 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:44,330 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:00:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-04T15:00:44,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T15:00:44,332 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:00:44,332 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:00:44,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:00:44,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324504420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:44,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:44,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:44,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:44,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324504430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:44,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:44,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:44,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T15:00:44,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-04T15:00:44,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:44,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:44,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:44,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:44,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:44,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:44,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324504630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:44,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-04T15:00:44,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:44,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:44,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:44,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:44,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:44,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324504639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:44,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:44,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:44,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T15:00:44,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/ebfbf5f55905415ab0cf940dbd688a8f 2024-12-04T15:00:44,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/fb4a882e81e54343a9e0c9802637f257 is 50, key is test_row_0/B:col10/1733324444273/Put/seqid=0 2024-12-04T15:00:44,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742307_1483 (size=12151) 2024-12-04T15:00:44,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/fb4a882e81e54343a9e0c9802637f257 2024-12-04T15:00:44,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/ac613810d12a4d308fa4ae6c9e268123 is 50, key is test_row_0/C:col10/1733324444273/Put/seqid=0 2024-12-04T15:00:44,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742308_1484 (size=12151) 2024-12-04T15:00:44,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-04T15:00:44,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:44,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:44,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:44,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:44,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:44,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-04T15:00:44,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:44,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:44,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:44,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:44,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:44,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:44,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:44,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324504940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:44,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:44,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:44,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:44,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:44,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324504950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:44,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:44,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:44,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:44,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T15:00:45,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-04T15:00:45,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:45,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:45,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:45,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:45,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:45,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/ac613810d12a4d308fa4ae6c9e268123 2024-12-04T15:00:45,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/ebfbf5f55905415ab0cf940dbd688a8f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/ebfbf5f55905415ab0cf940dbd688a8f 2024-12-04T15:00:45,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/ebfbf5f55905415ab0cf940dbd688a8f, entries=200, sequenceid=150, filesize=14.2 K 2024-12-04T15:00:45,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/fb4a882e81e54343a9e0c9802637f257 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/fb4a882e81e54343a9e0c9802637f257 2024-12-04T15:00:45,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/fb4a882e81e54343a9e0c9802637f257, entries=150, sequenceid=150, filesize=11.9 K 2024-12-04T15:00:45,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/ac613810d12a4d308fa4ae6c9e268123 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/ac613810d12a4d308fa4ae6c9e268123 2024-12-04T15:00:45,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/ac613810d12a4d308fa4ae6c9e268123, entries=150, sequenceid=150, filesize=11.9 K 2024-12-04T15:00:45,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 15ba391c303ef265123c46f23b7fbad1 in 849ms, sequenceid=150, compaction requested=true 2024-12-04T15:00:45,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:45,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:00:45,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:45,123 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T15:00:45,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:00:45,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:45,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:00:45,123 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T15:00:45,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:45,124 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55480 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T15:00:45,124 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T15:00:45,124 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/B is initiating minor compaction (all files) 2024-12-04T15:00:45,124 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/A is initiating minor compaction (all files) 2024-12-04T15:00:45,124 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/B in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:45,124 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/A in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:45,124 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/4a03a3ea6abb41d8be8e86edc4ab8c43, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/5afb594ccf5b4da4bac90b89dd37311c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/271c0bae28e04b958f57c4b872aee8ac, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/fb4a882e81e54343a9e0c9802637f257] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=47.3 K 2024-12-04T15:00:45,124 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/b5f4fcc78a0347cc8d72367d4458db84, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/36391914a4f4458d840b0907732e6306, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/da6ce47c7710483bb1732bcbc59cc188, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/ebfbf5f55905415ab0cf940dbd688a8f] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=54.2 K 2024-12-04T15:00:45,125 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a03a3ea6abb41d8be8e86edc4ab8c43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733324440535 2024-12-04T15:00:45,125 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5f4fcc78a0347cc8d72367d4458db84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733324440535 2024-12-04T15:00:45,125 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 5afb594ccf5b4da4bac90b89dd37311c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733324441852 2024-12-04T15:00:45,125 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36391914a4f4458d840b0907732e6306, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733324441839 2024-12-04T15:00:45,125 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 271c0bae28e04b958f57c4b872aee8ac, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733324441912 2024-12-04T15:00:45,126 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting da6ce47c7710483bb1732bcbc59cc188, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733324441912 2024-12-04T15:00:45,126 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting fb4a882e81e54343a9e0c9802637f257, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733324443099 2024-12-04T15:00:45,126 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebfbf5f55905415ab0cf940dbd688a8f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733324443099 2024-12-04T15:00:45,132 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#B#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:45,132 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/2ce2920b5f45476aa2f3dea0c27b9662 is 50, key is test_row_0/B:col10/1733324444273/Put/seqid=0 2024-12-04T15:00:45,134 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#A#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:45,134 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/fcb69ab2213248faa65fb75afdb41a64 is 50, key is test_row_0/A:col10/1733324444273/Put/seqid=0 2024-12-04T15:00:45,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742309_1485 (size=12493) 2024-12-04T15:00:45,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742310_1486 (size=12493) 2024-12-04T15:00:45,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-04T15:00:45,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:45,248 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T15:00:45,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:45,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:45,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:45,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:45,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:45,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/878e2a22c0be4f3cbccaaaca8d44b9dc is 50, key is test_row_0/A:col10/1733324444310/Put/seqid=0 2024-12-04T15:00:45,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742311_1487 (size=12151) 2024-12-04T15:00:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:45,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T15:00:45,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324505529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:45,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:45,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324505530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:45,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:45,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:45,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:45,541 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/2ce2920b5f45476aa2f3dea0c27b9662 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/2ce2920b5f45476aa2f3dea0c27b9662 2024-12-04T15:00:45,545 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/B of 15ba391c303ef265123c46f23b7fbad1 into 2ce2920b5f45476aa2f3dea0c27b9662(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:45,545 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:45,545 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/B, priority=12, startTime=1733324445123; duration=0sec 2024-12-04T15:00:45,545 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:45,545 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:B 2024-12-04T15:00:45,545 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T15:00:45,546 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T15:00:45,546 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/C is initiating minor compaction (all files) 2024-12-04T15:00:45,546 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/C in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:45,546 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/e3c10235dce34771829ca0ffe7d175f6, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/acb6cf2c8949423bbf63df6e0245d276, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/40727f1bae864819b1e2daed6ab241c2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/ac613810d12a4d308fa4ae6c9e268123] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=47.3 K 2024-12-04T15:00:45,546 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting e3c10235dce34771829ca0ffe7d175f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733324440535 2024-12-04T15:00:45,546 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting acb6cf2c8949423bbf63df6e0245d276, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733324441852 2024-12-04T15:00:45,547 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 40727f1bae864819b1e2daed6ab241c2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733324441912 2024-12-04T15:00:45,547 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting ac613810d12a4d308fa4ae6c9e268123, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733324443099 2024-12-04T15:00:45,550 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/fcb69ab2213248faa65fb75afdb41a64 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/fcb69ab2213248faa65fb75afdb41a64 2024-12-04T15:00:45,553 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#C#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:45,554 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/3c6e28725b0744ada6776737d7bdc220 is 50, key is test_row_0/C:col10/1733324444273/Put/seqid=0 2024-12-04T15:00:45,554 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/A of 15ba391c303ef265123c46f23b7fbad1 into fcb69ab2213248faa65fb75afdb41a64(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:45,554 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:45,554 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/A, priority=12, startTime=1733324445123; duration=0sec 2024-12-04T15:00:45,554 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:45,554 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:A 2024-12-04T15:00:45,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742312_1488 (size=12493) 2024-12-04T15:00:45,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:45,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324505640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:45,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:45,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:45,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:45,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:45,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324505640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:45,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:45,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:45,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:45,656 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/878e2a22c0be4f3cbccaaaca8d44b9dc 2024-12-04T15:00:45,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/3c52144cec364ed1b3560a1c080af4ef is 50, key is test_row_0/B:col10/1733324444310/Put/seqid=0 2024-12-04T15:00:45,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742313_1489 (size=12151) 2024-12-04T15:00:45,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:45,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324505850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:45,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324505850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:45,960 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/3c6e28725b0744ada6776737d7bdc220 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/3c6e28725b0744ada6776737d7bdc220 2024-12-04T15:00:45,964 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/C of 15ba391c303ef265123c46f23b7fbad1 into 3c6e28725b0744ada6776737d7bdc220(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:45,964 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:45,964 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/C, priority=12, startTime=1733324445123; duration=0sec 2024-12-04T15:00:45,964 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:45,964 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:C 2024-12-04T15:00:46,065 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/3c52144cec364ed1b3560a1c080af4ef 2024-12-04T15:00:46,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/26f5701bc908447298fa5a3c4cca1062 is 50, key is test_row_0/C:col10/1733324444310/Put/seqid=0 2024-12-04T15:00:46,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742314_1490 (size=12151) 2024-12-04T15:00:46,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324506160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:46,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:46,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:46,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:46,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324506160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:46,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:46,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:46,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:46,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45062 deadline: 1733324506170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:46,175 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 15ba391c303ef265123c46f23b7fbad1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:46,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:46,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:46,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:46,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1733324506190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:46,196 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_0' in 15ba391c303ef265123c46f23b7fbad1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:46,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1733324506200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:46,208 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_0' in 15ba391c303ef265123c46f23b7fbad1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:00:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:46,473 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/26f5701bc908447298fa5a3c4cca1062 2024-12-04T15:00:46,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/878e2a22c0be4f3cbccaaaca8d44b9dc as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/878e2a22c0be4f3cbccaaaca8d44b9dc 2024-12-04T15:00:46,480 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/878e2a22c0be4f3cbccaaaca8d44b9dc, entries=150, sequenceid=163, filesize=11.9 K 2024-12-04T15:00:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T15:00:46,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/3c52144cec364ed1b3560a1c080af4ef as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/3c52144cec364ed1b3560a1c080af4ef 2024-12-04T15:00:46,484 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/3c52144cec364ed1b3560a1c080af4ef, entries=150, sequenceid=163, filesize=11.9 K 2024-12-04T15:00:46,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/26f5701bc908447298fa5a3c4cca1062 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/26f5701bc908447298fa5a3c4cca1062 2024-12-04T15:00:46,487 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/26f5701bc908447298fa5a3c4cca1062, entries=150, sequenceid=163, filesize=11.9 K 2024-12-04T15:00:46,487 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 15ba391c303ef265123c46f23b7fbad1 in 1239ms, sequenceid=163, compaction requested=false 2024-12-04T15:00:46,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:46,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:46,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-04T15:00:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-04T15:00:46,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-04T15:00:46,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1570 sec 2024-12-04T15:00:46,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.1610 sec 2024-12-04T15:00:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:46,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T15:00:46,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:46,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:46,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:46,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:46,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:46,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:46,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/6e688d4c20294150bafbe2e006178273 is 50, key is test_row_0/A:col10/1733324446673/Put/seqid=0 2024-12-04T15:00:46,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742315_1491 (size=14541) 2024-12-04T15:00:46,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324506699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:46,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:46,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324506703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:46,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:46,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324506811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:46,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:46,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324506822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:47,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:47,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324507030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:47,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:47,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324507043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:47,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:47,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:47,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/6e688d4c20294150bafbe2e006178273 2024-12-04T15:00:47,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/29391b536e1f4c27a0cfe441e9ee10a6 is 50, key is test_row_0/B:col10/1733324446673/Put/seqid=0 2024-12-04T15:00:47,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742316_1492 (size=12151) 2024-12-04T15:00:47,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324507351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:47,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324507360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:47,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/29391b536e1f4c27a0cfe441e9ee10a6 2024-12-04T15:00:47,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/061f5af32d8b43bd8520ec4396bc6c13 is 50, key is test_row_0/C:col10/1733324446673/Put/seqid=0 2024-12-04T15:00:47,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742317_1493 (size=12151) 2024-12-04T15:00:47,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324507870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:47,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324507870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:47,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:47,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:47,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:47,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/061f5af32d8b43bd8520ec4396bc6c13 2024-12-04T15:00:47,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/6e688d4c20294150bafbe2e006178273 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/6e688d4c20294150bafbe2e006178273 2024-12-04T15:00:47,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/6e688d4c20294150bafbe2e006178273, entries=200, sequenceid=186, filesize=14.2 K 2024-12-04T15:00:47,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/29391b536e1f4c27a0cfe441e9ee10a6 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/29391b536e1f4c27a0cfe441e9ee10a6 2024-12-04T15:00:47,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/29391b536e1f4c27a0cfe441e9ee10a6, entries=150, sequenceid=186, filesize=11.9 K 2024-12-04T15:00:47,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/061f5af32d8b43bd8520ec4396bc6c13 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/061f5af32d8b43bd8520ec4396bc6c13 2024-12-04T15:00:47,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/061f5af32d8b43bd8520ec4396bc6c13, entries=150, sequenceid=186, filesize=11.9 K 2024-12-04T15:00:47,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 15ba391c303ef265123c46f23b7fbad1 in 1259ms, sequenceid=186, compaction requested=true 2024-12-04T15:00:47,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:47,933 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:47,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:00:47,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:47,933 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:47,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:00:47,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:47,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:00:47,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:47,934 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:47,934 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/B is initiating minor compaction (all files) 2024-12-04T15:00:47,934 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:47,934 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/A is initiating minor compaction (all files) 2024-12-04T15:00:47,934 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/B in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:47,934 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/A in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:47,934 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/2ce2920b5f45476aa2f3dea0c27b9662, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/3c52144cec364ed1b3560a1c080af4ef, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/29391b536e1f4c27a0cfe441e9ee10a6] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=35.9 K 2024-12-04T15:00:47,934 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/fcb69ab2213248faa65fb75afdb41a64, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/878e2a22c0be4f3cbccaaaca8d44b9dc, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/6e688d4c20294150bafbe2e006178273] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=38.3 K 2024-12-04T15:00:47,935 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ce2920b5f45476aa2f3dea0c27b9662, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733324443099 2024-12-04T15:00:47,935 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcb69ab2213248faa65fb75afdb41a64, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733324443099 2024-12-04T15:00:47,935 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c52144cec364ed1b3560a1c080af4ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733324444292 2024-12-04T15:00:47,935 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 878e2a22c0be4f3cbccaaaca8d44b9dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733324444292 2024-12-04T15:00:47,936 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e688d4c20294150bafbe2e006178273, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733324445515 2024-12-04T15:00:47,936 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 29391b536e1f4c27a0cfe441e9ee10a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733324445517 2024-12-04T15:00:47,943 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#B#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:47,943 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#A#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:47,943 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/689b03a35e764e399fc75f7a6fcc8de8 is 50, key is test_row_0/B:col10/1733324446673/Put/seqid=0 2024-12-04T15:00:47,943 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/02705c054f9544ec951041bf343e4af3 is 50, key is test_row_0/A:col10/1733324446673/Put/seqid=0 2024-12-04T15:00:47,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742318_1494 (size=12595) 2024-12-04T15:00:47,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742319_1495 (size=12595) 2024-12-04T15:00:47,967 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/02705c054f9544ec951041bf343e4af3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/02705c054f9544ec951041bf343e4af3 2024-12-04T15:00:47,974 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/A of 15ba391c303ef265123c46f23b7fbad1 into 02705c054f9544ec951041bf343e4af3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:47,974 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:47,974 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/A, priority=13, startTime=1733324447933; duration=0sec 2024-12-04T15:00:47,974 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:47,974 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:A 2024-12-04T15:00:47,974 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:47,975 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:47,975 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/C is initiating minor compaction (all files) 2024-12-04T15:00:47,975 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/C in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:47,975 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/3c6e28725b0744ada6776737d7bdc220, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/26f5701bc908447298fa5a3c4cca1062, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/061f5af32d8b43bd8520ec4396bc6c13] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=35.9 K 2024-12-04T15:00:47,975 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c6e28725b0744ada6776737d7bdc220, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733324443099 2024-12-04T15:00:47,975 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26f5701bc908447298fa5a3c4cca1062, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733324444292 2024-12-04T15:00:47,975 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 061f5af32d8b43bd8520ec4396bc6c13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733324445517 2024-12-04T15:00:47,979 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#C#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:47,979 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/019aa0104eac441bbc6644aa6b83716f is 50, key is test_row_0/C:col10/1733324446673/Put/seqid=0 2024-12-04T15:00:47,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742320_1496 (size=12595) 2024-12-04T15:00:48,367 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/689b03a35e764e399fc75f7a6fcc8de8 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/689b03a35e764e399fc75f7a6fcc8de8 2024-12-04T15:00:48,370 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/B of 15ba391c303ef265123c46f23b7fbad1 into 689b03a35e764e399fc75f7a6fcc8de8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:48,370 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:48,370 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/B, priority=13, startTime=1733324447933; duration=0sec 2024-12-04T15:00:48,370 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:48,370 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:B 2024-12-04T15:00:48,387 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/019aa0104eac441bbc6644aa6b83716f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/019aa0104eac441bbc6644aa6b83716f 2024-12-04T15:00:48,390 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/C of 15ba391c303ef265123c46f23b7fbad1 into 019aa0104eac441bbc6644aa6b83716f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:48,390 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:48,391 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/C, priority=13, startTime=1733324447934; duration=0sec 2024-12-04T15:00:48,391 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:48,391 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:C 2024-12-04T15:00:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T15:00:48,510 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:48,510 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:00:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-04T15:00:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T15:00:48,512 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:00:48,512 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:00:48,513 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:00:48,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T15:00:48,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-04T15:00:48,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:48,663 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T15:00:48,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:48,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:48,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:48,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:48,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:48,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:48,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/f331c3b0c35344e6b53decbaf9c1f16f is 50, key is test_row_0/A:col10/1733324446698/Put/seqid=0 2024-12-04T15:00:48,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742321_1497 (size=12151) 2024-12-04T15:00:48,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T15:00:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:48,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:48,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324508932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:48,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:48,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:48,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:48,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:48,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324508937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:48,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:48,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:48,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:49,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:49,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324509040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:49,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:49,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:49,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:49,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324509050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:49,070 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/f331c3b0c35344e6b53decbaf9c1f16f 2024-12-04T15:00:49,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/dc8ac03fc24a489db3b68d8f85bef25d is 50, key is test_row_0/B:col10/1733324446698/Put/seqid=0 2024-12-04T15:00:49,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742322_1498 (size=12151) 2024-12-04T15:00:49,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T15:00:49,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:49,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324509250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:49,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:49,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324509260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:49,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:49,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:49,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:49,484 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/dc8ac03fc24a489db3b68d8f85bef25d 2024-12-04T15:00:49,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/1a210f8edbe44632aba56087139888da is 50, key is test_row_0/C:col10/1733324446698/Put/seqid=0 2024-12-04T15:00:49,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742323_1499 (size=12151) 2024-12-04T15:00:49,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:49,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324509560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:49,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:49,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:49,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:49,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324509570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T15:00:49,897 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/1a210f8edbe44632aba56087139888da 2024-12-04T15:00:49,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/f331c3b0c35344e6b53decbaf9c1f16f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/f331c3b0c35344e6b53decbaf9c1f16f 2024-12-04T15:00:49,903 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/f331c3b0c35344e6b53decbaf9c1f16f, entries=150, sequenceid=201, filesize=11.9 K 2024-12-04T15:00:49,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/dc8ac03fc24a489db3b68d8f85bef25d as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/dc8ac03fc24a489db3b68d8f85bef25d 2024-12-04T15:00:49,907 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/dc8ac03fc24a489db3b68d8f85bef25d, entries=150, sequenceid=201, filesize=11.9 K 2024-12-04T15:00:49,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/1a210f8edbe44632aba56087139888da as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/1a210f8edbe44632aba56087139888da 2024-12-04T15:00:49,911 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/1a210f8edbe44632aba56087139888da, entries=150, sequenceid=201, filesize=11.9 K 2024-12-04T15:00:49,911 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 15ba391c303ef265123c46f23b7fbad1 in 1248ms, sequenceid=201, compaction requested=false 2024-12-04T15:00:49,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:49,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:49,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-04T15:00:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-04T15:00:49,914 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-04T15:00:49,914 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4010 sec 2024-12-04T15:00:49,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.4040 sec 2024-12-04T15:00:50,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-04T15:00:50,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:50,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:50,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:50,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:50,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:50,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:50,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/edbdad10583747e694db73c6aeaf55e3 is 50, key is test_row_0/A:col10/1733324448931/Put/seqid=0 2024-12-04T15:00:50,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742324_1500 (size=14541) 2024-12-04T15:00:50,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:50,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324510102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:50,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:50,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:50,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:50,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324510103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:50,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:50,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:50,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:50,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:50,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324510210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:50,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:50,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:50,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:50,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324510220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:50,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:50,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324510420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:50,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:50,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324510430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:50,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/edbdad10583747e694db73c6aeaf55e3 2024-12-04T15:00:50,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/147a10c42aca49818eecd775fc8ce5be is 50, key is test_row_0/B:col10/1733324448931/Put/seqid=0 2024-12-04T15:00:50,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742325_1501 (size=12151) 2024-12-04T15:00:50,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T15:00:50,660 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:50,660 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:00:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-04T15:00:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T15:00:50,662 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:00:50,662 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:00:50,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:00:50,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324510730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:50,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:50,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324510740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:50,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:50,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:50,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T15:00:50,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T15:00:50,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:50,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:50,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:50,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:50,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:50,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/147a10c42aca49818eecd775fc8ce5be 2024-12-04T15:00:50,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/be50e5c83d684c6c97cf605e38895e78 is 50, key is test_row_0/C:col10/1733324448931/Put/seqid=0 2024-12-04T15:00:50,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742326_1502 (size=12151) 2024-12-04T15:00:50,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T15:00:50,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:50,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:50,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:50,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:50,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:50,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T15:00:51,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T15:00:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:51,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:51,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:51,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:51,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:51,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324511250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:51,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324511250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:51,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T15:00:51,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:51,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:51,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:51,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:51,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T15:00:51,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/be50e5c83d684c6c97cf605e38895e78 2024-12-04T15:00:51,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/edbdad10583747e694db73c6aeaf55e3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/edbdad10583747e694db73c6aeaf55e3 2024-12-04T15:00:51,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/edbdad10583747e694db73c6aeaf55e3, entries=200, sequenceid=222, filesize=14.2 K 2024-12-04T15:00:51,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/147a10c42aca49818eecd775fc8ce5be as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/147a10c42aca49818eecd775fc8ce5be 2024-12-04T15:00:51,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/147a10c42aca49818eecd775fc8ce5be, entries=150, sequenceid=222, filesize=11.9 K 2024-12-04T15:00:51,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/be50e5c83d684c6c97cf605e38895e78 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/be50e5c83d684c6c97cf605e38895e78 2024-12-04T15:00:51,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/be50e5c83d684c6c97cf605e38895e78, entries=150, sequenceid=222, filesize=11.9 K 2024-12-04T15:00:51,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 15ba391c303ef265123c46f23b7fbad1 in 1248ms, sequenceid=222, compaction requested=true 2024-12-04T15:00:51,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:51,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:00:51,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:51,322 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:51,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:00:51,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:51,322 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:51,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:00:51,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:51,323 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:51,323 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/B is initiating minor compaction (all files) 2024-12-04T15:00:51,323 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/B in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:51,323 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/689b03a35e764e399fc75f7a6fcc8de8, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/dc8ac03fc24a489db3b68d8f85bef25d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/147a10c42aca49818eecd775fc8ce5be] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=36.0 K 2024-12-04T15:00:51,323 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:51,323 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 689b03a35e764e399fc75f7a6fcc8de8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733324445517 2024-12-04T15:00:51,323 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/A is initiating minor compaction (all files) 2024-12-04T15:00:51,323 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/A in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:51,324 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/02705c054f9544ec951041bf343e4af3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/f331c3b0c35344e6b53decbaf9c1f16f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/edbdad10583747e694db73c6aeaf55e3] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=38.4 K 2024-12-04T15:00:51,324 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting dc8ac03fc24a489db3b68d8f85bef25d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733324446693 2024-12-04T15:00:51,324 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02705c054f9544ec951041bf343e4af3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733324445517 2024-12-04T15:00:51,324 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 147a10c42aca49818eecd775fc8ce5be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733324448927 2024-12-04T15:00:51,324 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f331c3b0c35344e6b53decbaf9c1f16f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733324446693 2024-12-04T15:00:51,325 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting edbdad10583747e694db73c6aeaf55e3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733324448927 2024-12-04T15:00:51,330 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#A#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:51,331 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/f2783de965554b7faf6c7ac195478a85 is 50, key is test_row_0/A:col10/1733324448931/Put/seqid=0 2024-12-04T15:00:51,332 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#B#compaction#424 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:51,333 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/cc0ed4acda4847aea692a16615c7ecb2 is 50, key is test_row_0/B:col10/1733324448931/Put/seqid=0 2024-12-04T15:00:51,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742328_1504 (size=12697) 2024-12-04T15:00:51,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742327_1503 (size=12697) 2024-12-04T15:00:51,352 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/f2783de965554b7faf6c7ac195478a85 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/f2783de965554b7faf6c7ac195478a85 2024-12-04T15:00:51,352 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/cc0ed4acda4847aea692a16615c7ecb2 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/cc0ed4acda4847aea692a16615c7ecb2 2024-12-04T15:00:51,359 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/A of 15ba391c303ef265123c46f23b7fbad1 into f2783de965554b7faf6c7ac195478a85(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:51,359 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/B of 15ba391c303ef265123c46f23b7fbad1 into cc0ed4acda4847aea692a16615c7ecb2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:51,359 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:51,359 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:51,359 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/B, priority=13, startTime=1733324451322; duration=0sec 2024-12-04T15:00:51,359 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/A, priority=13, startTime=1733324451322; duration=0sec 2024-12-04T15:00:51,359 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:51,359 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:51,359 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:B 2024-12-04T15:00:51,359 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:A 2024-12-04T15:00:51,360 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:51,360 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:51,360 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/C is initiating minor compaction (all files) 2024-12-04T15:00:51,360 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/C in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:51,360 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/019aa0104eac441bbc6644aa6b83716f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/1a210f8edbe44632aba56087139888da, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/be50e5c83d684c6c97cf605e38895e78] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=36.0 K 2024-12-04T15:00:51,361 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 019aa0104eac441bbc6644aa6b83716f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733324445517 2024-12-04T15:00:51,361 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a210f8edbe44632aba56087139888da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733324446693 2024-12-04T15:00:51,361 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting be50e5c83d684c6c97cf605e38895e78, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733324448927 2024-12-04T15:00:51,366 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#C#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:51,366 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/7c8a4cbe550944f6965508334786ab0e is 50, key is test_row_0/C:col10/1733324448931/Put/seqid=0 2024-12-04T15:00:51,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742329_1505 (size=12697) 2024-12-04T15:00:51,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T15:00:51,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:51,425 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T15:00:51,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:51,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:51,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:51,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:51,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:51,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:51,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/4f39b4df3d8641ec82a63768b17b972d is 50, key is test_row_0/A:col10/1733324450101/Put/seqid=0 2024-12-04T15:00:51,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742330_1506 (size=9757) 2024-12-04T15:00:51,774 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/7c8a4cbe550944f6965508334786ab0e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/7c8a4cbe550944f6965508334786ab0e 2024-12-04T15:00:51,777 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/C of 15ba391c303ef265123c46f23b7fbad1 into 7c8a4cbe550944f6965508334786ab0e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:51,777 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:51,778 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/C, priority=13, startTime=1733324451322; duration=0sec 2024-12-04T15:00:51,778 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:51,778 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:C 2024-12-04T15:00:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T15:00:51,833 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/4f39b4df3d8641ec82a63768b17b972d 2024-12-04T15:00:51,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/b7096ecf65ae4eada743ecda94b7e78c is 50, key is test_row_0/B:col10/1733324450101/Put/seqid=0 2024-12-04T15:00:51,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742331_1507 (size=9757) 2024-12-04T15:00:52,242 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/b7096ecf65ae4eada743ecda94b7e78c 2024-12-04T15:00:52,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/f472cffd18744babbaaa00f56b0eddd8 is 50, key is test_row_0/C:col10/1733324450101/Put/seqid=0 2024-12-04T15:00:52,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742332_1508 (size=9757) 2024-12-04T15:00:52,253 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/f472cffd18744babbaaa00f56b0eddd8 2024-12-04T15:00:52,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/4f39b4df3d8641ec82a63768b17b972d as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/4f39b4df3d8641ec82a63768b17b972d 2024-12-04T15:00:52,261 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/4f39b4df3d8641ec82a63768b17b972d, entries=100, sequenceid=235, filesize=9.5 K 2024-12-04T15:00:52,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/b7096ecf65ae4eada743ecda94b7e78c as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/b7096ecf65ae4eada743ecda94b7e78c 2024-12-04T15:00:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:52,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:52,268 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/b7096ecf65ae4eada743ecda94b7e78c, entries=100, sequenceid=235, filesize=9.5 K 2024-12-04T15:00:52,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/f472cffd18744babbaaa00f56b0eddd8 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/f472cffd18744babbaaa00f56b0eddd8 2024-12-04T15:00:52,275 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/f472cffd18744babbaaa00f56b0eddd8, entries=100, sequenceid=235, filesize=9.5 K 2024-12-04T15:00:52,276 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=26.84 KB/27480 for 15ba391c303ef265123c46f23b7fbad1 in 851ms, sequenceid=235, compaction requested=false 2024-12-04T15:00:52,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:52,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:52,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-04T15:00:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-04T15:00:52,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-04T15:00:52,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6160 sec 2024-12-04T15:00:52,281 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.6200 sec 2024-12-04T15:00:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:52,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T15:00:52,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:52,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:52,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:52,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:52,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:52,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:52,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/8e8710b6a93545f084c8488d6012396d is 50, key is test_row_0/A:col10/1733324452273/Put/seqid=0 2024-12-04T15:00:52,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742333_1509 (size=16931) 2024-12-04T15:00:52,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324452319, max timestamp=1733324452344Num uniques 859; 2024-12-04T15:00:52,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T15:00:52,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324452319, max timestamp=1733324452344Num uniques 859; 2024-12-04T15:00:52,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T15:00:52,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324452319, max timestamp=1733324452344Num uniques 859; 2024-12-04T15:00:52,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T15:00:52,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324512438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:52,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324512443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:52,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:52,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:52,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:52,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:52,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:52,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:52,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324512560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:52,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:52,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:52,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324512560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:52,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:52,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:52,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:52,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/8e8710b6a93545f084c8488d6012396d 2024-12-04T15:00:52,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/57a253e34c3548ddb312d501cf733cc2 is 50, key is test_row_0/B:col10/1733324452273/Put/seqid=0 2024-12-04T15:00:52,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742334_1510 (size=12151) 2024-12-04T15:00:52,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324512781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:52,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324512781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:52,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:52,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:52,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:52,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:52,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:52,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T15:00:52,810 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:52,811 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:00:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-04T15:00:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T15:00:52,815 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:00:52,816 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:00:52,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:00:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T15:00:52,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-04T15:00:52,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:52,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:52,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:52,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:52,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:53,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/57a253e34c3548ddb312d501cf733cc2 2024-12-04T15:00:53,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:53,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324513101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:53,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:53,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:53,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324513101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:53,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:53,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:53,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:53,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:53,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:53,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/5cd6e791020743a29739589a4395647b is 50, key is test_row_0/C:col10/1733324452273/Put/seqid=0 2024-12-04T15:00:53,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-04T15:00:53,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:53,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:53,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:53,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:53,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:53,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T15:00:53,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742335_1511 (size=12151) 2024-12-04T15:00:53,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/5cd6e791020743a29739589a4395647b 2024-12-04T15:00:53,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/8e8710b6a93545f084c8488d6012396d as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/8e8710b6a93545f084c8488d6012396d 2024-12-04T15:00:53,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/8e8710b6a93545f084c8488d6012396d, entries=250, sequenceid=246, filesize=16.5 K 2024-12-04T15:00:53,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/57a253e34c3548ddb312d501cf733cc2 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/57a253e34c3548ddb312d501cf733cc2 2024-12-04T15:00:53,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/57a253e34c3548ddb312d501cf733cc2, entries=150, sequenceid=246, filesize=11.9 K 2024-12-04T15:00:53,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/5cd6e791020743a29739589a4395647b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/5cd6e791020743a29739589a4395647b 2024-12-04T15:00:53,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/5cd6e791020743a29739589a4395647b, entries=150, sequenceid=246, filesize=11.9 K 2024-12-04T15:00:53,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 15ba391c303ef265123c46f23b7fbad1 in 899ms, sequenceid=246, compaction requested=true 2024-12-04T15:00:53,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:53,182 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:53,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:00:53,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:53,182 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:53,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:00:53,183 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39385 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:53,183 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34605 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:53,183 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/A is initiating minor compaction (all files) 2024-12-04T15:00:53,183 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/B is initiating minor compaction (all files) 2024-12-04T15:00:53,183 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/B in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:53,183 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/A in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:53,183 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/cc0ed4acda4847aea692a16615c7ecb2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/b7096ecf65ae4eada743ecda94b7e78c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/57a253e34c3548ddb312d501cf733cc2] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=33.8 K 2024-12-04T15:00:53,183 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/f2783de965554b7faf6c7ac195478a85, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/4f39b4df3d8641ec82a63768b17b972d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/8e8710b6a93545f084c8488d6012396d] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=38.5 K 2024-12-04T15:00:53,183 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting cc0ed4acda4847aea692a16615c7ecb2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733324448927 2024-12-04T15:00:53,183 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2783de965554b7faf6c7ac195478a85, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733324448927 2024-12-04T15:00:53,184 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting b7096ecf65ae4eada743ecda94b7e78c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733324450097 2024-12-04T15:00:53,184 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f39b4df3d8641ec82a63768b17b972d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733324450097 2024-12-04T15:00:53,184 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e8710b6a93545f084c8488d6012396d, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733324452262 2024-12-04T15:00:53,184 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 57a253e34c3548ddb312d501cf733cc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733324452273 2024-12-04T15:00:53,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:53,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:00:53,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:53,191 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#B#compaction#432 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:53,191 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/32a31159226648b7a891665a02c9aab0 is 50, key is test_row_0/B:col10/1733324452273/Put/seqid=0 2024-12-04T15:00:53,191 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#A#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:53,192 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/832f428390ed470bb79949e3c67ee6a3 is 50, key is test_row_0/A:col10/1733324452273/Put/seqid=0 2024-12-04T15:00:53,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742336_1512 (size=12799) 2024-12-04T15:00:53,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742337_1513 (size=12799) 2024-12-04T15:00:53,226 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T15:00:53,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-04T15:00:53,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:53,279 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-04T15:00:53,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:53,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:53,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:53,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:53,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:53,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:53,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/707c2a2a8132469193a763d7c93b563c is 50, key is test_row_0/A:col10/1733324452441/Put/seqid=0 2024-12-04T15:00:53,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742338_1514 (size=12301) 2024-12-04T15:00:53,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T15:00:53,615 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/32a31159226648b7a891665a02c9aab0 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/32a31159226648b7a891665a02c9aab0 2024-12-04T15:00:53,622 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/832f428390ed470bb79949e3c67ee6a3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/832f428390ed470bb79949e3c67ee6a3 2024-12-04T15:00:53,629 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/B of 15ba391c303ef265123c46f23b7fbad1 into 32a31159226648b7a891665a02c9aab0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:53,629 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:53,629 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/B, priority=13, startTime=1733324453182; duration=0sec 2024-12-04T15:00:53,629 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:53,629 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:B 2024-12-04T15:00:53,629 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:53,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:53,630 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34605 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:53,630 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/C is initiating minor compaction (all files) 2024-12-04T15:00:53,630 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/C in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:53,630 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/7c8a4cbe550944f6965508334786ab0e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/f472cffd18744babbaaa00f56b0eddd8, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/5cd6e791020743a29739589a4395647b] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=33.8 K 2024-12-04T15:00:53,631 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c8a4cbe550944f6965508334786ab0e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733324448927 2024-12-04T15:00:53,631 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting f472cffd18744babbaaa00f56b0eddd8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733324450097 2024-12-04T15:00:53,631 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cd6e791020743a29739589a4395647b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733324452273 2024-12-04T15:00:53,634 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/A of 15ba391c303ef265123c46f23b7fbad1 into 832f428390ed470bb79949e3c67ee6a3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:53,634 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:53,634 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/A, priority=13, startTime=1733324453182; duration=0sec 2024-12-04T15:00:53,634 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:53,634 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:A 2024-12-04T15:00:53,653 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#C#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:53,653 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/d1aefb61c6d74f06a21d3c8e72b78b75 is 50, key is test_row_0/C:col10/1733324452273/Put/seqid=0 2024-12-04T15:00:53,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:53,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324513654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:53,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:53,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:53,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:53,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324513656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:53,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:53,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:53,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:53,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:53,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742339_1515 (size=12799) 2024-12-04T15:00:53,704 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/d1aefb61c6d74f06a21d3c8e72b78b75 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/d1aefb61c6d74f06a21d3c8e72b78b75 2024-12-04T15:00:53,710 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/C of 15ba391c303ef265123c46f23b7fbad1 into d1aefb61c6d74f06a21d3c8e72b78b75(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:53,710 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:53,710 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/C, priority=13, startTime=1733324453184; duration=0sec 2024-12-04T15:00:53,710 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:53,710 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:C 2024-12-04T15:00:53,716 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/707c2a2a8132469193a763d7c93b563c 2024-12-04T15:00:53,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/7ff9631448d34f86b14b16a49d9fe5ad is 50, key is test_row_0/B:col10/1733324452441/Put/seqid=0 2024-12-04T15:00:53,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742340_1516 (size=12301) 2024-12-04T15:00:53,744 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/7ff9631448d34f86b14b16a49d9fe5ad 2024-12-04T15:00:53,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/0bf1150ee1784c8390a4b6c8ba6b5cb3 is 50, key is test_row_0/C:col10/1733324452441/Put/seqid=0 2024-12-04T15:00:53,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742341_1517 (size=12301) 2024-12-04T15:00:53,768 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/0bf1150ee1784c8390a4b6c8ba6b5cb3 2024-12-04T15:00:53,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/707c2a2a8132469193a763d7c93b563c as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/707c2a2a8132469193a763d7c93b563c 2024-12-04T15:00:53,777 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/707c2a2a8132469193a763d7c93b563c, entries=150, sequenceid=282, filesize=12.0 K 2024-12-04T15:00:53,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/7ff9631448d34f86b14b16a49d9fe5ad as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/7ff9631448d34f86b14b16a49d9fe5ad 2024-12-04T15:00:53,780 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/7ff9631448d34f86b14b16a49d9fe5ad, entries=150, sequenceid=282, filesize=12.0 K 2024-12-04T15:00:53,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/0bf1150ee1784c8390a4b6c8ba6b5cb3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/0bf1150ee1784c8390a4b6c8ba6b5cb3 2024-12-04T15:00:53,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324513770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:53,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:53,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:53,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324513770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:53,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:53,784 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/0bf1150ee1784c8390a4b6c8ba6b5cb3, entries=150, sequenceid=282, filesize=12.0 K 2024-12-04T15:00:53,784 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 15ba391c303ef265123c46f23b7fbad1 in 505ms, sequenceid=282, compaction requested=false 2024-12-04T15:00:53,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:53,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:53,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-04T15:00:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-04T15:00:53,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-04T15:00:53,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 969 msec 2024-12-04T15:00:53,794 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 980 msec 2024-12-04T15:00:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T15:00:53,950 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:53,951 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:00:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-04T15:00:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T15:00:53,953 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:00:53,953 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:00:53,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:00:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:53,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T15:00:53,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:53,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:53,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:53,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:53,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:53,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:54,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/e0dd0c97824e48149ed57c471b63a555 is 50, key is test_row_0/A:col10/1733324453653/Put/seqid=0 2024-12-04T15:00:54,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742342_1518 (size=14741) 2024-12-04T15:00:54,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/e0dd0c97824e48149ed57c471b63a555 2024-12-04T15:00:54,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/b9274637587c4a7cb85d877501ecef05 is 50, key is test_row_0/B:col10/1733324453653/Put/seqid=0 2024-12-04T15:00:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T15:00:54,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742343_1519 (size=12301) 2024-12-04T15:00:54,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/b9274637587c4a7cb85d877501ecef05 2024-12-04T15:00:54,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/2c764490b93742f1ba659e05d8cb6e73 is 50, key is test_row_0/C:col10/1733324453653/Put/seqid=0 2024-12-04T15:00:54,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T15:00:54,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:54,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:54,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:54,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:54,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:54,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324454004, max timestamp=1733324454119Num uniques 859; 2024-12-04T15:00:54,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T15:00:54,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324454004, max timestamp=1733324454119Num uniques 859; 2024-12-04T15:00:54,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T15:00:54,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324454004, max timestamp=1733324454119Num uniques 859; 2024-12-04T15:00:54,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T15:00:54,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742344_1520 (size=12301) 2024-12-04T15:00:54,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/2c764490b93742f1ba659e05d8cb6e73 2024-12-04T15:00:54,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/e0dd0c97824e48149ed57c471b63a555 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/e0dd0c97824e48149ed57c471b63a555 2024-12-04T15:00:54,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/e0dd0c97824e48149ed57c471b63a555, entries=200, sequenceid=296, filesize=14.4 K 2024-12-04T15:00:54,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/b9274637587c4a7cb85d877501ecef05 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/b9274637587c4a7cb85d877501ecef05 2024-12-04T15:00:54,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/b9274637587c4a7cb85d877501ecef05, entries=150, sequenceid=296, filesize=12.0 K 2024-12-04T15:00:54,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/2c764490b93742f1ba659e05d8cb6e73 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/2c764490b93742f1ba659e05d8cb6e73 2024-12-04T15:00:54,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/2c764490b93742f1ba659e05d8cb6e73, entries=150, sequenceid=296, filesize=12.0 K 2024-12-04T15:00:54,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=80.51 KB/82440 for 15ba391c303ef265123c46f23b7fbad1 in 171ms, sequenceid=296, compaction requested=true 2024-12-04T15:00:54,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:54,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:00:54,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:54,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:00:54,167 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:54,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:54,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:00:54,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-04T15:00:54,167 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:54,168 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39841 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:54,168 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:54,168 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/A is initiating minor compaction (all files) 2024-12-04T15:00:54,168 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/B is initiating minor compaction (all files) 2024-12-04T15:00:54,168 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/B in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:54,168 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/32a31159226648b7a891665a02c9aab0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/7ff9631448d34f86b14b16a49d9fe5ad, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/b9274637587c4a7cb85d877501ecef05] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=36.5 K 2024-12-04T15:00:54,168 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/A in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:54,168 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/832f428390ed470bb79949e3c67ee6a3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/707c2a2a8132469193a763d7c93b563c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/e0dd0c97824e48149ed57c471b63a555] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=38.9 K 2024-12-04T15:00:54,168 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32a31159226648b7a891665a02c9aab0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733324452273 2024-12-04T15:00:54,168 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 832f428390ed470bb79949e3c67ee6a3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733324452273 2024-12-04T15:00:54,169 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ff9631448d34f86b14b16a49d9fe5ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733324452426 2024-12-04T15:00:54,169 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 707c2a2a8132469193a763d7c93b563c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733324452426 2024-12-04T15:00:54,169 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9274637587c4a7cb85d877501ecef05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733324453644 2024-12-04T15:00:54,169 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting e0dd0c97824e48149ed57c471b63a555, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733324453644 2024-12-04T15:00:54,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=87.22 KB heapSize=240.84 KB 2024-12-04T15:00:54,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:54,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:54,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:54,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:54,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:54,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:54,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/ee9f8c1c05704fb98488d6d42aa60e2f is 50, key is test_row_0/A:col10/1733324454138/Put/seqid=0 2024-12-04T15:00:54,207 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#A#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:54,207 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/6494defd02064889a530fa6fa36e736c is 50, key is test_row_0/A:col10/1733324453653/Put/seqid=0 2024-12-04T15:00:54,242 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#B#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:54,242 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/58ce25ea80304185884e1fdbba683eb9 is 50, key is test_row_0/B:col10/1733324453653/Put/seqid=0 2024-12-04T15:00:54,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742345_1521 (size=14741) 2024-12-04T15:00:54,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T15:00:54,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:54,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:54,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:54,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:54,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:54,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742346_1522 (size=13051) 2024-12-04T15:00:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T15:00:54,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742347_1523 (size=13051) 2024-12-04T15:00:54,278 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/58ce25ea80304185884e1fdbba683eb9 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/58ce25ea80304185884e1fdbba683eb9 2024-12-04T15:00:54,281 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/B of 15ba391c303ef265123c46f23b7fbad1 into 58ce25ea80304185884e1fdbba683eb9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:54,281 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:54,281 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/B, priority=13, startTime=1733324454167; duration=0sec 2024-12-04T15:00:54,281 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:54,281 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:B 2024-12-04T15:00:54,282 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:00:54,282 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:00:54,282 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/C is initiating minor compaction (all files) 2024-12-04T15:00:54,282 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/C in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:54,282 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/d1aefb61c6d74f06a21d3c8e72b78b75, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/0bf1150ee1784c8390a4b6c8ba6b5cb3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/2c764490b93742f1ba659e05d8cb6e73] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=36.5 K 2024-12-04T15:00:54,283 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1aefb61c6d74f06a21d3c8e72b78b75, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733324452273 2024-12-04T15:00:54,283 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bf1150ee1784c8390a4b6c8ba6b5cb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733324452426 2024-12-04T15:00:54,283 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c764490b93742f1ba659e05d8cb6e73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733324453644 2024-12-04T15:00:54,288 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#C#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:54,288 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/79a3bb47373449b99d670228d380e143 is 50, key is test_row_0/C:col10/1733324453653/Put/seqid=0 2024-12-04T15:00:54,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742348_1524 (size=13051) 2024-12-04T15:00:54,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324514285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:54,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:54,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324514289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:54,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:54,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:54,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:54,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:54,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:54,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324514400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:54,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324514400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:54,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T15:00:54,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:54,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:54,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:54,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:54,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:54,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T15:00:54,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:54,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:54,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:54,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:54,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T15:00:54,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324514610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:54,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324514610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:54,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:54,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:54,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:54,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:54,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/ee9f8c1c05704fb98488d6d42aa60e2f 2024-12-04T15:00:54,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/dd1e86e9b5bc44fc988684cc8fa7c5d1 is 50, key is test_row_0/B:col10/1733324454138/Put/seqid=0 2024-12-04T15:00:54,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742349_1525 (size=12301) 2024-12-04T15:00:54,673 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/6494defd02064889a530fa6fa36e736c as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/6494defd02064889a530fa6fa36e736c 2024-12-04T15:00:54,676 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/A of 15ba391c303ef265123c46f23b7fbad1 into 6494defd02064889a530fa6fa36e736c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:54,676 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:54,676 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/A, priority=13, startTime=1733324454167; duration=0sec 2024-12-04T15:00:54,677 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:54,677 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:A 2024-12-04T15:00:54,695 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/79a3bb47373449b99d670228d380e143 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/79a3bb47373449b99d670228d380e143 2024-12-04T15:00:54,698 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/C of 15ba391c303ef265123c46f23b7fbad1 into 79a3bb47373449b99d670228d380e143(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:54,698 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:54,698 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/C, priority=13, startTime=1733324454167; duration=0sec 2024-12-04T15:00:54,698 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:54,698 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:C 2024-12-04T15:00:54,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T15:00:54,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:54,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:54,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:54,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:54,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:54,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T15:00:54,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:54,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:54,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:54,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:54,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:54,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324514920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:54,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324514930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:55,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T15:00:55,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:55,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:55,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:55,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:55,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:55,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/dd1e86e9b5bc44fc988684cc8fa7c5d1 2024-12-04T15:00:55,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/59acf1dd137247198a59932b3c5f1e37 is 50, key is test_row_0/C:col10/1733324454138/Put/seqid=0 2024-12-04T15:00:55,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742350_1526 (size=12301) 2024-12-04T15:00:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T15:00:55,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T15:00:55,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:55,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:55,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:55,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:55,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:55,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T15:00:55,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:55,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:55,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:55,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:55,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:55,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:55,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324515430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:55,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:55,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:55,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324515450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:55,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/59acf1dd137247198a59932b3c5f1e37 2024-12-04T15:00:55,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T15:00:55,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:55,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:55,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/ee9f8c1c05704fb98488d6d42aa60e2f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/ee9f8c1c05704fb98488d6d42aa60e2f 2024-12-04T15:00:55,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:55,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:55,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:55,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/ee9f8c1c05704fb98488d6d42aa60e2f, entries=200, sequenceid=325, filesize=14.4 K 2024-12-04T15:00:55,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/dd1e86e9b5bc44fc988684cc8fa7c5d1 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/dd1e86e9b5bc44fc988684cc8fa7c5d1 2024-12-04T15:00:55,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/dd1e86e9b5bc44fc988684cc8fa7c5d1, entries=150, sequenceid=325, filesize=12.0 K 2024-12-04T15:00:55,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/59acf1dd137247198a59932b3c5f1e37 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/59acf1dd137247198a59932b3c5f1e37 2024-12-04T15:00:55,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/59acf1dd137247198a59932b3c5f1e37, entries=150, sequenceid=325, filesize=12.0 K 2024-12-04T15:00:55,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~240.80 KB/246576, currentSize=93.93 KB/96180 for 15ba391c303ef265123c46f23b7fbad1 in 1316ms, sequenceid=325, compaction requested=false 2024-12-04T15:00:55,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:55,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T15:00:55,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:55,634 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-04T15:00:55,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:55,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:55,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:55,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:55,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:55,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:55,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/258e3cebafde400b8efc01a799f1d89b is 50, key is test_row_0/A:col10/1733324454251/Put/seqid=0 2024-12-04T15:00:55,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742351_1527 (size=12301) 2024-12-04T15:00:55,642 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/258e3cebafde400b8efc01a799f1d89b 2024-12-04T15:00:55,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/e8cb7c3c675f46faa10de16cf01f5ca0 is 50, key is test_row_0/B:col10/1733324454251/Put/seqid=0 2024-12-04T15:00:55,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742352_1528 (size=12301) 2024-12-04T15:00:55,672 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/e8cb7c3c675f46faa10de16cf01f5ca0 2024-12-04T15:00:55,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/23eb81294adb42ce9c3825fc693c8e80 is 50, key is test_row_0/C:col10/1733324454251/Put/seqid=0 2024-12-04T15:00:55,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742353_1529 (size=12301) 2024-12-04T15:00:55,692 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/23eb81294adb42ce9c3825fc693c8e80 2024-12-04T15:00:55,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/258e3cebafde400b8efc01a799f1d89b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/258e3cebafde400b8efc01a799f1d89b 2024-12-04T15:00:55,700 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/258e3cebafde400b8efc01a799f1d89b, entries=150, sequenceid=345, filesize=12.0 K 2024-12-04T15:00:55,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/e8cb7c3c675f46faa10de16cf01f5ca0 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/e8cb7c3c675f46faa10de16cf01f5ca0 2024-12-04T15:00:55,703 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/e8cb7c3c675f46faa10de16cf01f5ca0, entries=150, sequenceid=345, filesize=12.0 K 2024-12-04T15:00:55,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/23eb81294adb42ce9c3825fc693c8e80 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/23eb81294adb42ce9c3825fc693c8e80 2024-12-04T15:00:55,713 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/23eb81294adb42ce9c3825fc693c8e80, entries=150, sequenceid=345, filesize=12.0 K 2024-12-04T15:00:55,714 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 15ba391c303ef265123c46f23b7fbad1 in 81ms, sequenceid=345, compaction requested=true 2024-12-04T15:00:55,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:55,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:55,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-04T15:00:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-04T15:00:55,716 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-04T15:00:55,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7610 sec 2024-12-04T15:00:55,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.7660 sec 2024-12-04T15:00:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T15:00:56,100 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:56,100 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:00:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-04T15:00:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T15:00:56,102 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:00:56,102 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:00:56,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:00:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T15:00:56,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-04T15:00:56,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:56,254 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-04T15:00:56,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:56,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:56,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:56,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:56,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:56,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:56,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/4b7f1d37f4f149aea7df9eaedb83f687 is 50, key is test_row_0/A:col10/1733324456247/Put/seqid=0 2024-12-04T15:00:56,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742354_1530 (size=14741) 2024-12-04T15:00:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:56,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:56,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324456283, max timestamp=1733324456312Num uniques 859; 2024-12-04T15:00:56,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324456283, max timestamp=1733324456312Num uniques 859; 2024-12-04T15:00:56,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T15:00:56,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T15:00:56,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324456283, max timestamp=1733324456312Num uniques 859; 2024-12-04T15:00:56,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T15:00:56,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733324456347, max timestamp=1733324456366Num uniques 1068; 2024-12-04T15:00:56,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733324456347, max timestamp=1733324456366Num uniques 1068; 2024-12-04T15:00:56,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733324456347, max timestamp=1733324456366Num uniques 1068; 2024-12-04T15:00:56,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-04T15:00:56,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-04T15:00:56,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-04T15:00:56,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T15:00:56,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45062 deadline: 1733324516428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:56,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8093, totalHeapSize=18648, min timestamp=1733324456413, max timestamp=1733324456443Num uniques 1127; 2024-12-04T15:00:56,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51603, new segment data size=8093, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-12-04T15:00:56,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8100, totalHeapSize=18648, min timestamp=1733324456413, max timestamp=1733324456443Num uniques 1127; 2024-12-04T15:00:56,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51610, new segment data size=8100, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-12-04T15:00:56,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733324456413, max timestamp=1733324456443Num uniques 1127; 2024-12-04T15:00:56,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51605, new segment data size=6870, suffix heap size=145176,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=150 2024-12-04T15:00:56,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10796, totalHeapSize=24784, min timestamp=1733324456486, max timestamp=1733324456501Num uniques 1036; 2024-12-04T15:00:56,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10796, totalHeapSize=24784, min timestamp=1733324456486, max timestamp=1733324456501Num uniques 1036; 2024-12-04T15:00:56,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47436, new segment data size=10796, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-04T15:00:56,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47436, new segment data size=10796, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-04T15:00:56,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10796, totalHeapSize=24784, min timestamp=1733324456486, max timestamp=1733324456501Num uniques 1009; 2024-12-04T15:00:56,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46211, new segment data size=10796, suffix heap size=132904,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=236 2024-12-04T15:00:56,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733324456532, max timestamp=1733324456548Num uniques 1095; 2024-12-04T15:00:56,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733324456532, max timestamp=1733324456548Num uniques 1095; 2024-12-04T15:00:56,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-12-04T15:00:56,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-12-04T15:00:56,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733324456532, max timestamp=1733324456548Num uniques 1095; 2024-12-04T15:00:56,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-12-04T15:00:56,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733324456575, max timestamp=1733324456599Num uniques 1054; 2024-12-04T15:00:56,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733324456575, max timestamp=1733324456599Num uniques 1054; 2024-12-04T15:00:56,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-12-04T15:00:56,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-12-04T15:00:56,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733324456575, max timestamp=1733324456599Num uniques 1054; 2024-12-04T15:00:56,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-12-04T15:00:56,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1733324516631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:56,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324516632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:56,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1733324516633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:56,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733324456628, max timestamp=1733324456639Num uniques 1113; 2024-12-04T15:00:56,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733324456628, max timestamp=1733324456639Num uniques 1113; 2024-12-04T15:00:56,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-12-04T15:00:56,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-12-04T15:00:56,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733324456628, max timestamp=1733324456639Num uniques 1113; 2024-12-04T15:00:56,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-12-04T15:00:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:56,668 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/4b7f1d37f4f149aea7df9eaedb83f687 2024-12-04T15:00:56,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/3542e931606540f9922d77f85cf28577 is 50, key is test_row_0/B:col10/1733324456247/Put/seqid=0 2024-12-04T15:00:56,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742355_1531 (size=12301) 2024-12-04T15:00:56,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45062 deadline: 1733324516712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:56,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733324456707, max timestamp=1733324456721Num uniques 1122; 2024-12-04T15:00:56,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733324456707, max timestamp=1733324456721Num uniques 1122; 2024-12-04T15:00:56,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-12-04T15:00:56,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-12-04T15:00:56,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733324456707, max timestamp=1733324456721Num uniques 1122; 2024-12-04T15:00:56,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-12-04T15:00:56,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T15:00:56,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733324456768, max timestamp=1733324456787Num uniques 1031; 2024-12-04T15:00:56,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-04T15:00:56,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733324456768, max timestamp=1733324456787Num uniques 1031; 2024-12-04T15:00:56,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-04T15:00:56,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733324456768, max timestamp=1733324456787Num uniques 1031; 2024-12-04T15:00:56,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-04T15:00:56,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1733324456810, max timestamp=1733324456834Num uniques 1040; 2024-12-04T15:00:56,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=15570, suffix heap size=136128,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=340 2024-12-04T15:00:56,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1733324456810, max timestamp=1733324456834Num uniques 1040; 2024-12-04T15:00:56,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=15570, suffix heap size=136128,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=340 2024-12-04T15:00:56,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1733324456810, max timestamp=1733324456834Num uniques 1040; 2024-12-04T15:00:56,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=15570, suffix heap size=136128,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=340 2024-12-04T15:00:56,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324516865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:56,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1733324516866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324516871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:56,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1733324456854, max timestamp=1733324456876Num uniques 1199; 2024-12-04T15:00:56,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=13694, suffix heap size=152664,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=299 2024-12-04T15:00:56,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1733324456854, max timestamp=1733324456876Num uniques 1199; 2024-12-04T15:00:56,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=13694, suffix heap size=152664,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=299 2024-12-04T15:00:56,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1733324456854, max timestamp=1733324456876Num uniques 1199; 2024-12-04T15:00:56,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=13694, suffix heap size=152664,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=299 2024-12-04T15:00:56,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1733324516953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:56,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45062 deadline: 1733324516957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:56,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324516980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:56,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1733324516980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:56,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:56,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324516990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:56,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:56,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:57,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1733324517070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:57,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45062 deadline: 1733324517070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,077 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/3542e931606540f9922d77f85cf28577 2024-12-04T15:00:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:57,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/33e4f2a4296d4ee492d1d891959a71c8 is 50, key is test_row_0/C:col10/1733324456247/Put/seqid=0 2024-12-04T15:00:57,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742356_1532 (size=12301) 2024-12-04T15:00:57,091 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/33e4f2a4296d4ee492d1d891959a71c8 2024-12-04T15:00:57,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/4b7f1d37f4f149aea7df9eaedb83f687 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/4b7f1d37f4f149aea7df9eaedb83f687 2024-12-04T15:00:57,097 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/4b7f1d37f4f149aea7df9eaedb83f687, entries=200, sequenceid=352, filesize=14.4 K 2024-12-04T15:00:57,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/3542e931606540f9922d77f85cf28577 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/3542e931606540f9922d77f85cf28577 2024-12-04T15:00:57,101 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/3542e931606540f9922d77f85cf28577, entries=150, sequenceid=352, filesize=12.0 K 2024-12-04T15:00:57,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/33e4f2a4296d4ee492d1d891959a71c8 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/33e4f2a4296d4ee492d1d891959a71c8 2024-12-04T15:00:57,104 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/33e4f2a4296d4ee492d1d891959a71c8, entries=150, sequenceid=352, filesize=12.0 K 2024-12-04T15:00:57,105 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=154.31 KB/158010 for 15ba391c303ef265123c46f23b7fbad1 in 851ms, sequenceid=352, compaction requested=true 2024-12-04T15:00:57,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:57,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:57,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-04T15:00:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-04T15:00:57,108 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-04T15:00:57,108 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0040 sec 2024-12-04T15:00:57,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.0080 sec 2024-12-04T15:00:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:57,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=161.02 KB heapSize=452.25 KB 2024-12-04T15:00:57,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:57,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733324456927, max timestamp=1733324457193Num uniques 1158; 2024-12-04T15:00:57,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T15:00:57,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:57,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=7230, suffix heap size=148400,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=158 2024-12-04T15:00:57,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:00:57,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:57,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T15:00:57,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/5889f842109a439fbf28f034ef49ea6f is 50, key is test_row_0/A:col10/1733324456936/Put/seqid=0 2024-12-04T15:00:57,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742357_1533 (size=17181) 2024-12-04T15:00:57,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/5889f842109a439fbf28f034ef49ea6f 2024-12-04T15:00:57,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/82241b1e24ef4ed490c40e64f8662837 is 50, key is test_row_0/B:col10/1733324456936/Put/seqid=0 2024-12-04T15:00:57,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742358_1534 (size=12301) 2024-12-04T15:00:57,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324517223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:57,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1733324517224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:57,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324517225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T15:00:57,241 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:00:57,241 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:00:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-04T15:00:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T15:00:57,243 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:00:57,243 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:00:57,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:00:57,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1733324517280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:57,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:57,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45062 deadline: 1733324517280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:57,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:57,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324517340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:57,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1733324517341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:57,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324517341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T15:00:57,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-04T15:00:57,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:57,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:57,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:57,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:57,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:57,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-04T15:00:57,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:57,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:57,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:57,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:57,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:57,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324517550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:57,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324517550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:57,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T15:00:57,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1733324517560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:57,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:57,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1733324517590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:57,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:57,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45062 deadline: 1733324517590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:57,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:57,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:57,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/82241b1e24ef4ed490c40e64f8662837 2024-12-04T15:00:57,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/a4f86a68bb0744a4856ff85cf618d515 is 50, key is test_row_0/C:col10/1733324456936/Put/seqid=0 2024-12-04T15:00:57,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742359_1535 (size=12301) 2024-12-04T15:00:57,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-04T15:00:57,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:57,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:57,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:57,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:57,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:57,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:57,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-04T15:00:57,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:57,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:57,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:57,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:57,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:57,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T15:00:57,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:57,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45010 deadline: 1733324517870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:57,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:57,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733324517870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:57,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:00:57,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1733324517880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:00:57,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:00:57,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., hostname=d214edae7bd2,36649,1733324304801, seqNum=2 because the exception is null or not the one we care about 2024-12-04T15:00:57,885 INFO [Thread-2049 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2049. 2024-12-04T15:00:57,885 DEBUG [Thread-2049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:00:57,885 DEBUG [Thread-2049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:57,886 DEBUG [Thread-2049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:57,886 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:00:57,894 INFO [Thread-2051 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2051. 2024-12-04T15:00:57,894 DEBUG [Thread-2051 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:00:57,894 DEBUG [Thread-2051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:57,894 DEBUG [Thread-2051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:57,894 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:00:57,897 INFO [Thread-2047 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2047. 2024-12-04T15:00:57,897 DEBUG [Thread-2047 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:00:57,897 DEBUG [Thread-2047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:57,897 DEBUG [Thread-2047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:57,897 INFO [Thread-2053 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2053. 2024-12-04T15:00:57,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:00:57,897 DEBUG [Thread-2053 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:00:57,897 DEBUG [Thread-2053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:57,897 DEBUG [Thread-2053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:57,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:00:57,903 INFO [Thread-2055 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2055. 2024-12-04T15:00:57,904 DEBUG [Thread-2055 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:00:57,904 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:57,904 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:57,904 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:00:58,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-04T15:00:58,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:58,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:58,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:58,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:58,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:58,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:58,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/a4f86a68bb0744a4856ff85cf618d515 2024-12-04T15:00:58,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/5889f842109a439fbf28f034ef49ea6f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/5889f842109a439fbf28f034ef49ea6f 2024-12-04T15:00:58,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/5889f842109a439fbf28f034ef49ea6f, entries=250, sequenceid=563, filesize=16.8 K 2024-12-04T15:00:58,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/82241b1e24ef4ed490c40e64f8662837 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/82241b1e24ef4ed490c40e64f8662837 2024-12-04T15:00:58,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/82241b1e24ef4ed490c40e64f8662837, entries=150, sequenceid=563, filesize=12.0 K 2024-12-04T15:00:58,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/a4f86a68bb0744a4856ff85cf618d515 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/a4f86a68bb0744a4856ff85cf618d515 2024-12-04T15:00:58,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/a4f86a68bb0744a4856ff85cf618d515, entries=150, sequenceid=563, filesize=12.0 K 2024-12-04T15:00:58,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~123.00 KB/125950, heapSize ~343.48 KB/351728, currentSize=60.38 KB/61830 for 15ba391c303ef265123c46f23b7fbad1 in 845ms, sequenceid=563, compaction requested=true 2024-12-04T15:00:58,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:58,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:00:58,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:58,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:00:58,040 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T15:00:58,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:58,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15ba391c303ef265123c46f23b7fbad1:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:00:58,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:58,041 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T15:00:58,041 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 72015 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T15:00:58,041 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62255 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T15:00:58,041 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/B is initiating minor compaction (all files) 2024-12-04T15:00:58,041 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/A is initiating minor compaction (all files) 2024-12-04T15:00:58,041 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/A in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:58,041 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/B in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:58,042 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/6494defd02064889a530fa6fa36e736c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/ee9f8c1c05704fb98488d6d42aa60e2f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/258e3cebafde400b8efc01a799f1d89b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/4b7f1d37f4f149aea7df9eaedb83f687, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/5889f842109a439fbf28f034ef49ea6f] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=70.3 K 2024-12-04T15:00:58,042 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/58ce25ea80304185884e1fdbba683eb9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/dd1e86e9b5bc44fc988684cc8fa7c5d1, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/e8cb7c3c675f46faa10de16cf01f5ca0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/3542e931606540f9922d77f85cf28577, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/82241b1e24ef4ed490c40e64f8662837] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=60.8 K 2024-12-04T15:00:58,042 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6494defd02064889a530fa6fa36e736c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733324453644 2024-12-04T15:00:58,042 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 58ce25ea80304185884e1fdbba683eb9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733324453644 2024-12-04T15:00:58,042 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee9f8c1c05704fb98488d6d42aa60e2f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1733324454138 2024-12-04T15:00:58,042 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting dd1e86e9b5bc44fc988684cc8fa7c5d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1733324454138 2024-12-04T15:00:58,042 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 258e3cebafde400b8efc01a799f1d89b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733324454251 2024-12-04T15:00:58,042 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting e8cb7c3c675f46faa10de16cf01f5ca0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733324454251 2024-12-04T15:00:58,042 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b7f1d37f4f149aea7df9eaedb83f687, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733324456240 2024-12-04T15:00:58,042 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 3542e931606540f9922d77f85cf28577, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733324456242 2024-12-04T15:00:58,043 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5889f842109a439fbf28f034ef49ea6f, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1733324456927 2024-12-04T15:00:58,043 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 82241b1e24ef4ed490c40e64f8662837, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1733324456927 2024-12-04T15:00:58,050 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#B#compaction#456 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:58,051 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/f65f8aa608a54d22a3414642bcadedcd is 50, key is test_row_0/B:col10/1733324456936/Put/seqid=0 2024-12-04T15:00:58,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742360_1536 (size=13221) 2024-12-04T15:00:58,062 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#A#compaction#457 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:58,062 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/deca23d0ab6b442c94809d5f33d0bb20 is 50, key is test_row_0/A:col10/1733324456936/Put/seqid=0 2024-12-04T15:00:58,067 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/f65f8aa608a54d22a3414642bcadedcd as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/f65f8aa608a54d22a3414642bcadedcd 2024-12-04T15:00:58,072 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/B of 15ba391c303ef265123c46f23b7fbad1 into f65f8aa608a54d22a3414642bcadedcd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:58,072 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:58,072 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/B, priority=11, startTime=1733324458040; duration=0sec 2024-12-04T15:00:58,072 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:00:58,072 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:B 2024-12-04T15:00:58,072 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T15:00:58,073 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62255 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T15:00:58,073 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 15ba391c303ef265123c46f23b7fbad1/C is initiating minor compaction (all files) 2024-12-04T15:00:58,073 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15ba391c303ef265123c46f23b7fbad1/C in TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:58,073 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/79a3bb47373449b99d670228d380e143, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/59acf1dd137247198a59932b3c5f1e37, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/23eb81294adb42ce9c3825fc693c8e80, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/33e4f2a4296d4ee492d1d891959a71c8, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/a4f86a68bb0744a4856ff85cf618d515] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp, totalSize=60.8 K 2024-12-04T15:00:58,074 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 79a3bb47373449b99d670228d380e143, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733324453644 2024-12-04T15:00:58,074 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 59acf1dd137247198a59932b3c5f1e37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1733324454138 2024-12-04T15:00:58,074 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 23eb81294adb42ce9c3825fc693c8e80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733324454251 2024-12-04T15:00:58,074 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 33e4f2a4296d4ee492d1d891959a71c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733324456242 2024-12-04T15:00:58,075 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting a4f86a68bb0744a4856ff85cf618d515, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1733324456927 2024-12-04T15:00:58,106 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15ba391c303ef265123c46f23b7fbad1#C#compaction#458 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:00:58,106 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/54e197c7a0da433da52a68be59b49d84 is 50, key is test_row_0/C:col10/1733324456936/Put/seqid=0 2024-12-04T15:00:58,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:00:58,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-04T15:00:58,113 INFO [Thread-2040 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2040. 2024-12-04T15:00:58,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:58,113 DEBUG [Thread-2040 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:00:58,113 DEBUG [Thread-2040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:58,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:58,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:58,113 DEBUG [Thread-2040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:58,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:58,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:58,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:58,113 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:00:58,114 INFO [Thread-2038 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2038. 2024-12-04T15:00:58,114 DEBUG [Thread-2038 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:00:58,114 DEBUG [Thread-2038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:58,114 DEBUG [Thread-2038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:58,114 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:00:58,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742361_1537 (size=13221) 2024-12-04T15:00:58,124 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/deca23d0ab6b442c94809d5f33d0bb20 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/deca23d0ab6b442c94809d5f33d0bb20 2024-12-04T15:00:58,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/76703193993a461eb21fc973157acc71 is 50, key is test_row_0/A:col10/1733324458110/Put/seqid=0 2024-12-04T15:00:58,129 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/A of 15ba391c303ef265123c46f23b7fbad1 into deca23d0ab6b442c94809d5f33d0bb20(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:58,129 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:58,129 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/A, priority=11, startTime=1733324458040; duration=0sec 2024-12-04T15:00:58,129 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:58,129 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:A 2024-12-04T15:00:58,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742362_1538 (size=13221) 2024-12-04T15:00:58,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742363_1539 (size=12301) 2024-12-04T15:00:58,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=578 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/76703193993a461eb21fc973157acc71 2024-12-04T15:00:58,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/03c5433dfdfe4d44b6aa7652a6b2713a is 50, key is test_row_0/B:col10/1733324458110/Put/seqid=0 2024-12-04T15:00:58,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-04T15:00:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:58,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:58,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:58,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:58,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742364_1540 (size=12301) 2024-12-04T15:00:58,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=578 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/03c5433dfdfe4d44b6aa7652a6b2713a 2024-12-04T15:00:58,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/0dcbdf80604c49e689aec394b7dd2b79 is 50, key is test_row_0/C:col10/1733324458110/Put/seqid=0 2024-12-04T15:00:58,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742365_1541 (size=12301) 2024-12-04T15:00:58,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-04T15:00:58,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:58,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:58,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:58,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:58,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T15:00:58,382 INFO [Thread-2036 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2036. 2024-12-04T15:00:58,382 INFO [Thread-2042 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2042. 2024-12-04T15:00:58,383 DEBUG [Thread-2042 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:00:58,383 DEBUG [Thread-2036 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:00:58,383 DEBUG [Thread-2042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:58,383 DEBUG [Thread-2036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:58,383 DEBUG [Thread-2036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:58,383 DEBUG [Thread-2042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:58,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:00:58,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:00:58,392 INFO [Thread-2044 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2044. 2024-12-04T15:00:58,392 DEBUG [Thread-2044 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:00:58,392 DEBUG [Thread-2044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:58,392 DEBUG [Thread-2044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:00:58,392 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:00:58,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-04T15:00:58,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:58,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. as already flushing 2024-12-04T15:00:58,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:58,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:58,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:58,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:00:58,547 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/54e197c7a0da433da52a68be59b49d84 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/54e197c7a0da433da52a68be59b49d84 2024-12-04T15:00:58,551 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 15ba391c303ef265123c46f23b7fbad1/C of 15ba391c303ef265123c46f23b7fbad1 into 54e197c7a0da433da52a68be59b49d84(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:00:58,551 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:58,551 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1., storeName=15ba391c303ef265123c46f23b7fbad1/C, priority=11, startTime=1733324458040; duration=0sec 2024-12-04T15:00:58,551 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:00:58,551 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15ba391c303ef265123c46f23b7fbad1:C 2024-12-04T15:00:58,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=578 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/0dcbdf80604c49e689aec394b7dd2b79 2024-12-04T15:00:58,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/76703193993a461eb21fc973157acc71 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/76703193993a461eb21fc973157acc71 2024-12-04T15:00:58,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/76703193993a461eb21fc973157acc71, entries=150, sequenceid=578, filesize=12.0 K 2024-12-04T15:00:58,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/03c5433dfdfe4d44b6aa7652a6b2713a as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/03c5433dfdfe4d44b6aa7652a6b2713a 2024-12-04T15:00:58,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/03c5433dfdfe4d44b6aa7652a6b2713a, entries=150, sequenceid=578, filesize=12.0 K 2024-12-04T15:00:58,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/0dcbdf80604c49e689aec394b7dd2b79 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/0dcbdf80604c49e689aec394b7dd2b79 2024-12-04T15:00:58,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/0dcbdf80604c49e689aec394b7dd2b79, entries=150, sequenceid=578, filesize=12.0 K 2024-12-04T15:00:58,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=20.13 KB/20610 for 15ba391c303ef265123c46f23b7fbad1 in 504ms, sequenceid=578, compaction requested=false 2024-12-04T15:00:58,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:58,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-04T15:00:58,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:58,625 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 15ba391c303ef265123c46f23b7fbad1 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-04T15:00:58,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=A 2024-12-04T15:00:58,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:58,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=B 2024-12-04T15:00:58,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:58,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15ba391c303ef265123c46f23b7fbad1, store=C 2024-12-04T15:00:58,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:00:58,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/fef79a643436486b8d335de1b996ba80 is 50, key is test_row_1/A:col10/1733324458381/Put/seqid=0 2024-12-04T15:00:58,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742366_1542 (size=9857) 2024-12-04T15:00:59,032 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/fef79a643436486b8d335de1b996ba80 2024-12-04T15:00:59,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/567c8d74b8a34837a7e5ec3513fa5c2e is 50, key is test_row_1/B:col10/1733324458381/Put/seqid=0 2024-12-04T15:00:59,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742367_1543 (size=9857) 2024-12-04T15:00:59,044 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/567c8d74b8a34837a7e5ec3513fa5c2e 2024-12-04T15:00:59,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/be94a9d3dde44ed7b1b0e0f30921378b is 50, key is test_row_1/C:col10/1733324458381/Put/seqid=0 2024-12-04T15:00:59,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742368_1544 (size=9857) 2024-12-04T15:00:59,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T15:00:59,457 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/be94a9d3dde44ed7b1b0e0f30921378b 2024-12-04T15:00:59,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/A/fef79a643436486b8d335de1b996ba80 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/fef79a643436486b8d335de1b996ba80 2024-12-04T15:00:59,464 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/fef79a643436486b8d335de1b996ba80, entries=100, sequenceid=586, filesize=9.6 K 2024-12-04T15:00:59,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/B/567c8d74b8a34837a7e5ec3513fa5c2e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/567c8d74b8a34837a7e5ec3513fa5c2e 2024-12-04T15:00:59,468 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/567c8d74b8a34837a7e5ec3513fa5c2e, entries=100, sequenceid=586, filesize=9.6 K 2024-12-04T15:00:59,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/.tmp/C/be94a9d3dde44ed7b1b0e0f30921378b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/be94a9d3dde44ed7b1b0e0f30921378b 2024-12-04T15:00:59,473 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/be94a9d3dde44ed7b1b0e0f30921378b, entries=100, sequenceid=586, filesize=9.6 K 2024-12-04T15:00:59,474 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 15ba391c303ef265123c46f23b7fbad1 in 849ms, sequenceid=586, compaction requested=true 2024-12-04T15:00:59,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 15ba391c303ef265123c46f23b7fbad1: 2024-12-04T15:00:59,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:00:59,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-04T15:00:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-04T15:00:59,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-04T15:00:59,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2320 sec 2024-12-04T15:00:59,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.2360 sec 2024-12-04T15:01:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T15:01:01,411 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:01:01,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T15:01:01,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 165 2024-12-04T15:01:01,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-04T15:01:01,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-04T15:01:01,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 151 2024-12-04T15:01:01,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-12-04T15:01:01,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T15:01:01,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T15:01:01,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1605 2024-12-04T15:01:01,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4810 rows 2024-12-04T15:01:01,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1604 2024-12-04T15:01:01,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4811 rows 2024-12-04T15:01:01,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1610 2024-12-04T15:01:01,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4825 rows 2024-12-04T15:01:01,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1602 2024-12-04T15:01:01,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4804 rows 2024-12-04T15:01:01,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1614 2024-12-04T15:01:01,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4840 rows 2024-12-04T15:01:01,411 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T15:01:01,411 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T15:01:01,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:01,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:01,411 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:01:01,412 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T15:01:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T15:01:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-04T15:01:01,414 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324461414"}]},"ts":"1733324461414"} 2024-12-04T15:01:01,415 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T15:01:01,415 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T15:01:01,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T15:01:01,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=15ba391c303ef265123c46f23b7fbad1, UNASSIGN}] 2024-12-04T15:01:01,417 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=15ba391c303ef265123c46f23b7fbad1, UNASSIGN 2024-12-04T15:01:01,418 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=15ba391c303ef265123c46f23b7fbad1, regionState=CLOSING, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:01,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=15ba391c303ef265123c46f23b7fbad1, UNASSIGN because future has completed 2024-12-04T15:01:01,419 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T15:01:01,419 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801}] 2024-12-04T15:01:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-04T15:01:01,572 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:01:01,572 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T15:01:01,572 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing 15ba391c303ef265123c46f23b7fbad1, disabling compactions & flushes 2024-12-04T15:01:01,572 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:01:01,572 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:01:01,572 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. after waiting 0 ms 2024-12-04T15:01:01,572 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:01:01,572 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/a5946126bb9849e099defd46ec47f273, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/e215e39554eb4fafbbd78428b290eb47, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/9588e4d3d81c42048b75fd7ec2c9d3af, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/be8b4f37a4c645bc97a55791301047c0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/6b4769bf52304c8792191940a67b37bc, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/55b6d3fb71544c4e99af027507487f63, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/b5f4fcc78a0347cc8d72367d4458db84, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/36391914a4f4458d840b0907732e6306, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/da6ce47c7710483bb1732bcbc59cc188, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/ebfbf5f55905415ab0cf940dbd688a8f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/fcb69ab2213248faa65fb75afdb41a64, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/878e2a22c0be4f3cbccaaaca8d44b9dc, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/6e688d4c20294150bafbe2e006178273, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/02705c054f9544ec951041bf343e4af3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/f331c3b0c35344e6b53decbaf9c1f16f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/edbdad10583747e694db73c6aeaf55e3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/f2783de965554b7faf6c7ac195478a85, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/4f39b4df3d8641ec82a63768b17b972d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/8e8710b6a93545f084c8488d6012396d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/832f428390ed470bb79949e3c67ee6a3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/707c2a2a8132469193a763d7c93b563c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/e0dd0c97824e48149ed57c471b63a555, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/6494defd02064889a530fa6fa36e736c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/ee9f8c1c05704fb98488d6d42aa60e2f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/258e3cebafde400b8efc01a799f1d89b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/4b7f1d37f4f149aea7df9eaedb83f687, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/5889f842109a439fbf28f034ef49ea6f] to archive 2024-12-04T15:01:01,574 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T15:01:01,575 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/a5946126bb9849e099defd46ec47f273 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/a5946126bb9849e099defd46ec47f273 2024-12-04T15:01:01,576 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/e215e39554eb4fafbbd78428b290eb47 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/e215e39554eb4fafbbd78428b290eb47 2024-12-04T15:01:01,578 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/9588e4d3d81c42048b75fd7ec2c9d3af to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/9588e4d3d81c42048b75fd7ec2c9d3af 2024-12-04T15:01:01,579 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/be8b4f37a4c645bc97a55791301047c0 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/be8b4f37a4c645bc97a55791301047c0 2024-12-04T15:01:01,580 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/6b4769bf52304c8792191940a67b37bc to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/6b4769bf52304c8792191940a67b37bc 2024-12-04T15:01:01,581 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/55b6d3fb71544c4e99af027507487f63 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/55b6d3fb71544c4e99af027507487f63 2024-12-04T15:01:01,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/b5f4fcc78a0347cc8d72367d4458db84 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/b5f4fcc78a0347cc8d72367d4458db84 2024-12-04T15:01:01,583 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/36391914a4f4458d840b0907732e6306 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/36391914a4f4458d840b0907732e6306 2024-12-04T15:01:01,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/da6ce47c7710483bb1732bcbc59cc188 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/da6ce47c7710483bb1732bcbc59cc188 2024-12-04T15:01:01,585 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/ebfbf5f55905415ab0cf940dbd688a8f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/ebfbf5f55905415ab0cf940dbd688a8f 2024-12-04T15:01:01,586 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/fcb69ab2213248faa65fb75afdb41a64 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/fcb69ab2213248faa65fb75afdb41a64 2024-12-04T15:01:01,588 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/878e2a22c0be4f3cbccaaaca8d44b9dc to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/878e2a22c0be4f3cbccaaaca8d44b9dc 2024-12-04T15:01:01,588 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/6e688d4c20294150bafbe2e006178273 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/6e688d4c20294150bafbe2e006178273 2024-12-04T15:01:01,590 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/02705c054f9544ec951041bf343e4af3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/02705c054f9544ec951041bf343e4af3 2024-12-04T15:01:01,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/f331c3b0c35344e6b53decbaf9c1f16f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/f331c3b0c35344e6b53decbaf9c1f16f 2024-12-04T15:01:01,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/edbdad10583747e694db73c6aeaf55e3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/edbdad10583747e694db73c6aeaf55e3 2024-12-04T15:01:01,593 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/f2783de965554b7faf6c7ac195478a85 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/f2783de965554b7faf6c7ac195478a85 2024-12-04T15:01:01,595 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/4f39b4df3d8641ec82a63768b17b972d to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/4f39b4df3d8641ec82a63768b17b972d 2024-12-04T15:01:01,596 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/8e8710b6a93545f084c8488d6012396d to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/8e8710b6a93545f084c8488d6012396d 2024-12-04T15:01:01,597 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/832f428390ed470bb79949e3c67ee6a3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/832f428390ed470bb79949e3c67ee6a3 2024-12-04T15:01:01,598 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/707c2a2a8132469193a763d7c93b563c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/707c2a2a8132469193a763d7c93b563c 2024-12-04T15:01:01,600 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/e0dd0c97824e48149ed57c471b63a555 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/e0dd0c97824e48149ed57c471b63a555 2024-12-04T15:01:01,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/6494defd02064889a530fa6fa36e736c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/6494defd02064889a530fa6fa36e736c 2024-12-04T15:01:01,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/ee9f8c1c05704fb98488d6d42aa60e2f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/ee9f8c1c05704fb98488d6d42aa60e2f 2024-12-04T15:01:01,603 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/258e3cebafde400b8efc01a799f1d89b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/258e3cebafde400b8efc01a799f1d89b 2024-12-04T15:01:01,606 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/4b7f1d37f4f149aea7df9eaedb83f687 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/4b7f1d37f4f149aea7df9eaedb83f687 2024-12-04T15:01:01,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/5889f842109a439fbf28f034ef49ea6f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/5889f842109a439fbf28f034ef49ea6f 2024-12-04T15:01:01,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/e01a0b55da9845318cb2b4c637698ac0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/c379ce69dce24048b91cb2bb91e89e7f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/cc80b514be0445279efe426a0d334bca, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/20aec5b168ac4915a3104204c9893fb5, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/dee450952f824f41a1b680dd63b4c7f4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/4a03a3ea6abb41d8be8e86edc4ab8c43, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/62e5a881889b41819dd907998c029f22, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/5afb594ccf5b4da4bac90b89dd37311c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/271c0bae28e04b958f57c4b872aee8ac, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/2ce2920b5f45476aa2f3dea0c27b9662, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/fb4a882e81e54343a9e0c9802637f257, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/3c52144cec364ed1b3560a1c080af4ef, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/689b03a35e764e399fc75f7a6fcc8de8, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/29391b536e1f4c27a0cfe441e9ee10a6, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/dc8ac03fc24a489db3b68d8f85bef25d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/cc0ed4acda4847aea692a16615c7ecb2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/147a10c42aca49818eecd775fc8ce5be, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/b7096ecf65ae4eada743ecda94b7e78c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/32a31159226648b7a891665a02c9aab0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/57a253e34c3548ddb312d501cf733cc2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/7ff9631448d34f86b14b16a49d9fe5ad, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/58ce25ea80304185884e1fdbba683eb9, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/b9274637587c4a7cb85d877501ecef05, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/dd1e86e9b5bc44fc988684cc8fa7c5d1, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/e8cb7c3c675f46faa10de16cf01f5ca0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/3542e931606540f9922d77f85cf28577, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/82241b1e24ef4ed490c40e64f8662837] to archive 2024-12-04T15:01:01,614 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T15:01:01,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/e01a0b55da9845318cb2b4c637698ac0 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/e01a0b55da9845318cb2b4c637698ac0 2024-12-04T15:01:01,617 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/c379ce69dce24048b91cb2bb91e89e7f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/c379ce69dce24048b91cb2bb91e89e7f 2024-12-04T15:01:01,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/cc80b514be0445279efe426a0d334bca to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/cc80b514be0445279efe426a0d334bca 2024-12-04T15:01:01,619 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/20aec5b168ac4915a3104204c9893fb5 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/20aec5b168ac4915a3104204c9893fb5 2024-12-04T15:01:01,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/dee450952f824f41a1b680dd63b4c7f4 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/dee450952f824f41a1b680dd63b4c7f4 2024-12-04T15:01:01,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/4a03a3ea6abb41d8be8e86edc4ab8c43 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/4a03a3ea6abb41d8be8e86edc4ab8c43 2024-12-04T15:01:01,628 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/62e5a881889b41819dd907998c029f22 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/62e5a881889b41819dd907998c029f22 2024-12-04T15:01:01,629 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/5afb594ccf5b4da4bac90b89dd37311c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/5afb594ccf5b4da4bac90b89dd37311c 2024-12-04T15:01:01,630 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/271c0bae28e04b958f57c4b872aee8ac to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/271c0bae28e04b958f57c4b872aee8ac 2024-12-04T15:01:01,631 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/2ce2920b5f45476aa2f3dea0c27b9662 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/2ce2920b5f45476aa2f3dea0c27b9662 2024-12-04T15:01:01,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/fb4a882e81e54343a9e0c9802637f257 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/fb4a882e81e54343a9e0c9802637f257 2024-12-04T15:01:01,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/3c52144cec364ed1b3560a1c080af4ef to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/3c52144cec364ed1b3560a1c080af4ef 2024-12-04T15:01:01,633 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/689b03a35e764e399fc75f7a6fcc8de8 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/689b03a35e764e399fc75f7a6fcc8de8 2024-12-04T15:01:01,634 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/29391b536e1f4c27a0cfe441e9ee10a6 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/29391b536e1f4c27a0cfe441e9ee10a6 2024-12-04T15:01:01,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/dc8ac03fc24a489db3b68d8f85bef25d to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/dc8ac03fc24a489db3b68d8f85bef25d 2024-12-04T15:01:01,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/cc0ed4acda4847aea692a16615c7ecb2 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/cc0ed4acda4847aea692a16615c7ecb2 2024-12-04T15:01:01,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/147a10c42aca49818eecd775fc8ce5be to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/147a10c42aca49818eecd775fc8ce5be 2024-12-04T15:01:01,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/b7096ecf65ae4eada743ecda94b7e78c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/b7096ecf65ae4eada743ecda94b7e78c 2024-12-04T15:01:01,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/32a31159226648b7a891665a02c9aab0 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/32a31159226648b7a891665a02c9aab0 2024-12-04T15:01:01,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/57a253e34c3548ddb312d501cf733cc2 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/57a253e34c3548ddb312d501cf733cc2 2024-12-04T15:01:01,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/7ff9631448d34f86b14b16a49d9fe5ad to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/7ff9631448d34f86b14b16a49d9fe5ad 2024-12-04T15:01:01,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/58ce25ea80304185884e1fdbba683eb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/58ce25ea80304185884e1fdbba683eb9 2024-12-04T15:01:01,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/b9274637587c4a7cb85d877501ecef05 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/b9274637587c4a7cb85d877501ecef05 2024-12-04T15:01:01,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/dd1e86e9b5bc44fc988684cc8fa7c5d1 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/dd1e86e9b5bc44fc988684cc8fa7c5d1 2024-12-04T15:01:01,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/e8cb7c3c675f46faa10de16cf01f5ca0 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/e8cb7c3c675f46faa10de16cf01f5ca0 2024-12-04T15:01:01,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/3542e931606540f9922d77f85cf28577 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/3542e931606540f9922d77f85cf28577 2024-12-04T15:01:01,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/82241b1e24ef4ed490c40e64f8662837 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/82241b1e24ef4ed490c40e64f8662837 2024-12-04T15:01:01,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/379a7d2565f547adbda831707800be61, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/592e1268a1e14bdc8e0fa24767059214, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/6947e91cb1a6449c9caf6d7787a3fabd, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/75fcff6ff1e34560ad835e89d4a9b031, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/05730b1c73df415cae157fffebb34623, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/e3c10235dce34771829ca0ffe7d175f6, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/c561f1d255f2419abe56b618fc68a7ff, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/acb6cf2c8949423bbf63df6e0245d276, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/40727f1bae864819b1e2daed6ab241c2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/3c6e28725b0744ada6776737d7bdc220, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/ac613810d12a4d308fa4ae6c9e268123, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/26f5701bc908447298fa5a3c4cca1062, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/019aa0104eac441bbc6644aa6b83716f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/061f5af32d8b43bd8520ec4396bc6c13, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/1a210f8edbe44632aba56087139888da, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/7c8a4cbe550944f6965508334786ab0e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/be50e5c83d684c6c97cf605e38895e78, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/f472cffd18744babbaaa00f56b0eddd8, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/d1aefb61c6d74f06a21d3c8e72b78b75, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/5cd6e791020743a29739589a4395647b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/0bf1150ee1784c8390a4b6c8ba6b5cb3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/79a3bb47373449b99d670228d380e143, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/2c764490b93742f1ba659e05d8cb6e73, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/59acf1dd137247198a59932b3c5f1e37, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/23eb81294adb42ce9c3825fc693c8e80, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/33e4f2a4296d4ee492d1d891959a71c8, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/a4f86a68bb0744a4856ff85cf618d515] to archive 2024-12-04T15:01:01,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T15:01:01,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/379a7d2565f547adbda831707800be61 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/379a7d2565f547adbda831707800be61 2024-12-04T15:01:01,653 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/592e1268a1e14bdc8e0fa24767059214 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/592e1268a1e14bdc8e0fa24767059214 2024-12-04T15:01:01,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/6947e91cb1a6449c9caf6d7787a3fabd to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/6947e91cb1a6449c9caf6d7787a3fabd 2024-12-04T15:01:01,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/75fcff6ff1e34560ad835e89d4a9b031 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/75fcff6ff1e34560ad835e89d4a9b031 2024-12-04T15:01:01,656 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/05730b1c73df415cae157fffebb34623 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/05730b1c73df415cae157fffebb34623 2024-12-04T15:01:01,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/e3c10235dce34771829ca0ffe7d175f6 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/e3c10235dce34771829ca0ffe7d175f6 2024-12-04T15:01:01,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/c561f1d255f2419abe56b618fc68a7ff to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/c561f1d255f2419abe56b618fc68a7ff 2024-12-04T15:01:01,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/acb6cf2c8949423bbf63df6e0245d276 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/acb6cf2c8949423bbf63df6e0245d276 2024-12-04T15:01:01,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/40727f1bae864819b1e2daed6ab241c2 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/40727f1bae864819b1e2daed6ab241c2 2024-12-04T15:01:01,660 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/3c6e28725b0744ada6776737d7bdc220 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/3c6e28725b0744ada6776737d7bdc220 2024-12-04T15:01:01,661 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/ac613810d12a4d308fa4ae6c9e268123 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/ac613810d12a4d308fa4ae6c9e268123 2024-12-04T15:01:01,661 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/26f5701bc908447298fa5a3c4cca1062 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/26f5701bc908447298fa5a3c4cca1062 2024-12-04T15:01:01,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/019aa0104eac441bbc6644aa6b83716f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/019aa0104eac441bbc6644aa6b83716f 2024-12-04T15:01:01,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/061f5af32d8b43bd8520ec4396bc6c13 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/061f5af32d8b43bd8520ec4396bc6c13 2024-12-04T15:01:01,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/1a210f8edbe44632aba56087139888da to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/1a210f8edbe44632aba56087139888da 2024-12-04T15:01:01,664 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/7c8a4cbe550944f6965508334786ab0e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/7c8a4cbe550944f6965508334786ab0e 2024-12-04T15:01:01,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/be50e5c83d684c6c97cf605e38895e78 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/be50e5c83d684c6c97cf605e38895e78 2024-12-04T15:01:01,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/f472cffd18744babbaaa00f56b0eddd8 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/f472cffd18744babbaaa00f56b0eddd8 2024-12-04T15:01:01,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/d1aefb61c6d74f06a21d3c8e72b78b75 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/d1aefb61c6d74f06a21d3c8e72b78b75 2024-12-04T15:01:01,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/5cd6e791020743a29739589a4395647b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/5cd6e791020743a29739589a4395647b 2024-12-04T15:01:01,667 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/0bf1150ee1784c8390a4b6c8ba6b5cb3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/0bf1150ee1784c8390a4b6c8ba6b5cb3 2024-12-04T15:01:01,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/79a3bb47373449b99d670228d380e143 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/79a3bb47373449b99d670228d380e143 2024-12-04T15:01:01,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/2c764490b93742f1ba659e05d8cb6e73 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/2c764490b93742f1ba659e05d8cb6e73 2024-12-04T15:01:01,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/59acf1dd137247198a59932b3c5f1e37 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/59acf1dd137247198a59932b3c5f1e37 2024-12-04T15:01:01,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/23eb81294adb42ce9c3825fc693c8e80 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/23eb81294adb42ce9c3825fc693c8e80 2024-12-04T15:01:01,670 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/33e4f2a4296d4ee492d1d891959a71c8 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/33e4f2a4296d4ee492d1d891959a71c8 2024-12-04T15:01:01,671 DEBUG [StoreCloser-TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/a4f86a68bb0744a4856ff85cf618d515 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/a4f86a68bb0744a4856ff85cf618d515 2024-12-04T15:01:01,679 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/recovered.edits/589.seqid, newMaxSeqId=589, maxSeqId=1 2024-12-04T15:01:01,680 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1. 2024-12-04T15:01:01,680 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for 15ba391c303ef265123c46f23b7fbad1: Waiting for close lock at 1733324461572Running coprocessor pre-close hooks at 1733324461572Disabling compacts and flushes for region at 1733324461572Disabling writes for close at 1733324461572Writing region close event to WAL at 1733324461672 (+100 ms)Running coprocessor post-close hooks at 1733324461680 (+8 ms)Closed at 1733324461680 2024-12-04T15:01:01,682 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed 15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:01:01,682 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=15ba391c303ef265123c46f23b7fbad1, regionState=CLOSED 2024-12-04T15:01:01,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 because future has completed 2024-12-04T15:01:01,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-04T15:01:01,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure 15ba391c303ef265123c46f23b7fbad1, server=d214edae7bd2,36649,1733324304801 in 265 msec 2024-12-04T15:01:01,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-12-04T15:01:01,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=15ba391c303ef265123c46f23b7fbad1, UNASSIGN in 269 msec 2024-12-04T15:01:01,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-04T15:01:01,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 271 msec 2024-12-04T15:01:01,689 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324461689"}]},"ts":"1733324461689"} 2024-12-04T15:01:01,691 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T15:01:01,691 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T15:01:01,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 280 msec 2024-12-04T15:01:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-04T15:01:01,730 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:01:01,731 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T15:01:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:01:01,733 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:01:01,733 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:01:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-04T15:01:01,735 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:01:01,737 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A, FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B, FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C, FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/recovered.edits] 2024-12-04T15:01:01,739 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/76703193993a461eb21fc973157acc71 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/76703193993a461eb21fc973157acc71 2024-12-04T15:01:01,739 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/deca23d0ab6b442c94809d5f33d0bb20 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/deca23d0ab6b442c94809d5f33d0bb20 2024-12-04T15:01:01,741 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/fef79a643436486b8d335de1b996ba80 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/A/fef79a643436486b8d335de1b996ba80 2024-12-04T15:01:01,743 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/03c5433dfdfe4d44b6aa7652a6b2713a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/03c5433dfdfe4d44b6aa7652a6b2713a 2024-12-04T15:01:01,744 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/567c8d74b8a34837a7e5ec3513fa5c2e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/567c8d74b8a34837a7e5ec3513fa5c2e 2024-12-04T15:01:01,745 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/f65f8aa608a54d22a3414642bcadedcd to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/B/f65f8aa608a54d22a3414642bcadedcd 2024-12-04T15:01:01,747 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/0dcbdf80604c49e689aec394b7dd2b79 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/0dcbdf80604c49e689aec394b7dd2b79 2024-12-04T15:01:01,748 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/54e197c7a0da433da52a68be59b49d84 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/54e197c7a0da433da52a68be59b49d84 2024-12-04T15:01:01,749 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/be94a9d3dde44ed7b1b0e0f30921378b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/C/be94a9d3dde44ed7b1b0e0f30921378b 2024-12-04T15:01:01,751 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/recovered.edits/589.seqid to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1/recovered.edits/589.seqid 2024-12-04T15:01:01,751 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/15ba391c303ef265123c46f23b7fbad1 2024-12-04T15:01:01,751 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T15:01:01,753 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:01:01,755 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T15:01:01,757 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T15:01:01,758 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:01:01,758 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T15:01:01,758 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733324461758"}]},"ts":"9223372036854775807"} 2024-12-04T15:01:01,760 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T15:01:01,761 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 15ba391c303ef265123c46f23b7fbad1, NAME => 'TestAcidGuarantees,,1733324435677.15ba391c303ef265123c46f23b7fbad1.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T15:01:01,761 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T15:01:01,761 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733324461761"}]},"ts":"9223372036854775807"} 2024-12-04T15:01:01,762 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T15:01:01,763 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:01:01,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-12-04T15:01:01,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-04T15:01:01,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T15:01:01,840 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:01:01,848 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 249) - Thread LEAK? -, OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=440 (was 497), ProcessCount=11 (was 11), AvailableMemoryMB=9558 (was 8454) - AvailableMemoryMB LEAK? - 2024-12-04T15:01:01,856 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=440, ProcessCount=11, AvailableMemoryMB=9558 2024-12-04T15:01:01,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T15:01:01,857 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:01:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T15:01:01,859 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T15:01:01,859 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:01,859 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-12-04T15:01:01,860 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T15:01:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-04T15:01:01,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742369_1545 (size=1061) 2024-12-04T15:01:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-04T15:01:02,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-04T15:01:02,266 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee 2024-12-04T15:01:02,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742370_1546 (size=53) 2024-12-04T15:01:02,274 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:01:02,274 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9bc7edf6120a83a9451ae3a6357eaeb9, disabling compactions & flushes 2024-12-04T15:01:02,274 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:02,274 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:02,274 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. after waiting 0 ms 2024-12-04T15:01:02,274 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:02,274 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:02,274 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9bc7edf6120a83a9451ae3a6357eaeb9: Waiting for close lock at 1733324462274Disabling compacts and flushes for region at 1733324462274Disabling writes for close at 1733324462274Writing region close event to WAL at 1733324462274Closed at 1733324462274 2024-12-04T15:01:02,275 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T15:01:02,276 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733324462276"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733324462276"}]},"ts":"1733324462276"} 2024-12-04T15:01:02,278 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T15:01:02,279 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T15:01:02,279 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324462279"}]},"ts":"1733324462279"} 2024-12-04T15:01:02,280 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T15:01:02,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9bc7edf6120a83a9451ae3a6357eaeb9, ASSIGN}] 2024-12-04T15:01:02,282 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9bc7edf6120a83a9451ae3a6357eaeb9, ASSIGN 2024-12-04T15:01:02,282 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9bc7edf6120a83a9451ae3a6357eaeb9, ASSIGN; state=OFFLINE, location=d214edae7bd2,36649,1733324304801; forceNewPlan=false, retain=false 2024-12-04T15:01:02,433 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=9bc7edf6120a83a9451ae3a6357eaeb9, regionState=OPENING, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:02,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9bc7edf6120a83a9451ae3a6357eaeb9, ASSIGN because future has completed 2024-12-04T15:01:02,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801}] 2024-12-04T15:01:02,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-04T15:01:02,593 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:02,593 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} 2024-12-04T15:01:02,593 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:02,594 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:01:02,594 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:02,594 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:02,596 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:02,598 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T15:01:02,598 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bc7edf6120a83a9451ae3a6357eaeb9 columnFamilyName A 2024-12-04T15:01:02,598 DEBUG [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:02,598 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] regionserver.HStore(327): Store=9bc7edf6120a83a9451ae3a6357eaeb9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:01:02,598 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:02,599 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T15:01:02,600 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bc7edf6120a83a9451ae3a6357eaeb9 columnFamilyName B 2024-12-04T15:01:02,600 DEBUG [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:02,600 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] regionserver.HStore(327): Store=9bc7edf6120a83a9451ae3a6357eaeb9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:01:02,600 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:02,601 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T15:01:02,601 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bc7edf6120a83a9451ae3a6357eaeb9 columnFamilyName C 2024-12-04T15:01:02,602 DEBUG [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:02,602 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] regionserver.HStore(327): Store=9bc7edf6120a83a9451ae3a6357eaeb9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:01:02,602 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:02,602 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:02,603 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:02,603 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:02,604 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:02,604 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:02,604 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T15:01:02,605 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:02,607 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T15:01:02,607 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened 9bc7edf6120a83a9451ae3a6357eaeb9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75020177, jitterRate=0.11788775026798248}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T15:01:02,607 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:02,608 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for 9bc7edf6120a83a9451ae3a6357eaeb9: Running coprocessor pre-open hook at 1733324462594Writing region info on filesystem at 1733324462594Initializing all the Stores at 1733324462595 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324462595Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324462596 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324462596Cleaning up temporary data from old regions at 1733324462604 (+8 ms)Running coprocessor post-open hooks at 1733324462607 (+3 ms)Region opened successfully at 1733324462608 (+1 ms) 2024-12-04T15:01:02,609 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., pid=140, masterSystemTime=1733324462590 2024-12-04T15:01:02,610 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:02,610 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:02,611 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=9bc7edf6120a83a9451ae3a6357eaeb9, regionState=OPEN, openSeqNum=2, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:02,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 because future has completed 2024-12-04T15:01:02,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-04T15:01:02,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 in 178 msec 2024-12-04T15:01:02,618 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-04T15:01:02,618 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9bc7edf6120a83a9451ae3a6357eaeb9, ASSIGN in 336 msec 2024-12-04T15:01:02,619 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T15:01:02,619 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324462619"}]},"ts":"1733324462619"} 2024-12-04T15:01:02,620 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T15:01:02,621 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T15:01:02,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 764 msec 2024-12-04T15:01:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-04T15:01:03,001 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:01:03,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1847540, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:01:03,003 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:01:03,004 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:01:03,004 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:01:03,004 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:01:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ea8e92b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:01:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:01:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:03,007 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49492, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:01:03,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@683ee30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,008 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:01:03,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:01:03,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:01:03,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:01:03,012 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T15:01:03,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is d214edae7bd2,39449,1733324304115 2024-12-04T15:01:03,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@784215e6 2024-12-04T15:01:03,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T15:01:03,013 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49498, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T15:01:03,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T15:01:03,016 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:01:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-04T15:01:03,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742371_1547 (size=1097) 2024-12-04T15:01:03,428 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-04T15:01:03,428 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-04T15:01:03,431 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T15:01:03,432 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9bc7edf6120a83a9451ae3a6357eaeb9, REOPEN/MOVE}] 2024-12-04T15:01:03,433 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9bc7edf6120a83a9451ae3a6357eaeb9, REOPEN/MOVE 2024-12-04T15:01:03,434 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=9bc7edf6120a83a9451ae3a6357eaeb9, regionState=CLOSING, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:03,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9bc7edf6120a83a9451ae3a6357eaeb9, REOPEN/MOVE because future has completed 2024-12-04T15:01:03,435 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T15:01:03,435 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801}] 2024-12-04T15:01:03,588 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:03,588 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T15:01:03,588 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 9bc7edf6120a83a9451ae3a6357eaeb9, disabling compactions & flushes 2024-12-04T15:01:03,588 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:03,588 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:03,588 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. after waiting 0 ms 2024-12-04T15:01:03,588 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:03,594 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-04T15:01:03,595 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:03,595 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 9bc7edf6120a83a9451ae3a6357eaeb9: Waiting for close lock at 1733324463588Running coprocessor pre-close hooks at 1733324463588Disabling compacts and flushes for region at 1733324463588Disabling writes for close at 1733324463588Writing region close event to WAL at 1733324463589 (+1 ms)Running coprocessor post-close hooks at 1733324463594 (+5 ms)Closed at 1733324463595 (+1 ms) 2024-12-04T15:01:03,595 WARN [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: 9bc7edf6120a83a9451ae3a6357eaeb9 to self. 2024-12-04T15:01:03,597 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:03,598 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=9bc7edf6120a83a9451ae3a6357eaeb9, regionState=CLOSED 2024-12-04T15:01:03,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 because future has completed 2024-12-04T15:01:03,603 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-04T15:01:03,603 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 in 165 msec 2024-12-04T15:01:03,604 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9bc7edf6120a83a9451ae3a6357eaeb9, REOPEN/MOVE; state=CLOSED, location=d214edae7bd2,36649,1733324304801; forceNewPlan=false, retain=true 2024-12-04T15:01:03,754 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=9bc7edf6120a83a9451ae3a6357eaeb9, regionState=OPENING, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:03,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9bc7edf6120a83a9451ae3a6357eaeb9, REOPEN/MOVE because future has completed 2024-12-04T15:01:03,757 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801}] 2024-12-04T15:01:03,914 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:03,915 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} 2024-12-04T15:01:03,915 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:03,915 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:01:03,916 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:03,916 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:03,919 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:03,919 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T15:01:03,919 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bc7edf6120a83a9451ae3a6357eaeb9 columnFamilyName A 2024-12-04T15:01:03,921 DEBUG [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:03,921 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] regionserver.HStore(327): Store=9bc7edf6120a83a9451ae3a6357eaeb9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:01:03,921 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:03,922 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T15:01:03,922 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bc7edf6120a83a9451ae3a6357eaeb9 columnFamilyName B 2024-12-04T15:01:03,922 DEBUG [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:03,923 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] regionserver.HStore(327): Store=9bc7edf6120a83a9451ae3a6357eaeb9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:01:03,923 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:03,923 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T15:01:03,923 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bc7edf6120a83a9451ae3a6357eaeb9 columnFamilyName C 2024-12-04T15:01:03,923 DEBUG [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:03,924 INFO [StoreOpener-9bc7edf6120a83a9451ae3a6357eaeb9-1 {}] regionserver.HStore(327): Store=9bc7edf6120a83a9451ae3a6357eaeb9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:01:03,924 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:03,924 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:03,925 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:03,926 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:03,927 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:03,927 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:03,927 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T15:01:03,929 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:03,929 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened 9bc7edf6120a83a9451ae3a6357eaeb9; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59579575, jitterRate=-0.11219514906406403}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T15:01:03,929 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:03,930 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for 9bc7edf6120a83a9451ae3a6357eaeb9: Running coprocessor pre-open hook at 1733324463916Writing region info on filesystem at 1733324463916Initializing all the Stores at 1733324463918 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324463918Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324463918Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324463918Cleaning up temporary data from old regions at 1733324463927 (+9 ms)Running coprocessor post-open hooks at 1733324463929 (+2 ms)Region opened successfully at 1733324463930 (+1 ms) 2024-12-04T15:01:03,930 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., pid=145, masterSystemTime=1733324463909 2024-12-04T15:01:03,932 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:03,932 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:03,932 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=9bc7edf6120a83a9451ae3a6357eaeb9, regionState=OPEN, openSeqNum=5, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:03,933 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39449 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=d214edae7bd2,36649,1733324304801, table=TestAcidGuarantees, region=9bc7edf6120a83a9451ae3a6357eaeb9. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-04T15:01:03,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 because future has completed 2024-12-04T15:01:03,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-12-04T15:01:03,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 in 177 msec 2024-12-04T15:01:03,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-04T15:01:03,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9bc7edf6120a83a9451ae3a6357eaeb9, REOPEN/MOVE in 503 msec 2024-12-04T15:01:03,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-04T15:01:03,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 507 msec 2024-12-04T15:01:03,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 923 msec 2024-12-04T15:01:03,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-04T15:01:03,940 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:01:03,941 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68e6069e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,941 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:01:03,941 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:01:03,942 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:01:03,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:01:03,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:01:03,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43ef8f64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:01:03,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:01:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:03,943 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:01:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35ad17b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604e524e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,944 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:01:03,945 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:01:03,945 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:01:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:01:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:01:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f844354, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:01:03,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:01:03,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:03,946 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:01:03,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1457b09a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f36d46f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,948 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:01:03,948 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:01:03,948 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:01:03,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:01:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:01:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31ef54be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:01:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:01:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:03,950 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:01:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ae4c2f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e5e61df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,951 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:01:03,951 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:01:03,952 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:01:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:01:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:01:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aa21d9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:01:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:01:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:03,953 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:01:03,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@332892b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4885acdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,955 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:01:03,955 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:01:03,955 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:01:03,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:01:03,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:01:03,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d8b2344, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:01:03,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:01:03,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:03,956 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:01:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f4de190, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@473abea5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,957 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:01:03,958 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:01:03,958 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:01:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:01:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:01:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23714982, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:01:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:01:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:03,959 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:01:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c666db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,960 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@533d9664, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,960 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:01:03,960 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:01:03,961 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:01:03,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:01:03,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:01:03,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4253bb0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:01:03,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:01:03,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:03,962 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:01:03,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@102bcf5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,962 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@383c5fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,962 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:01:03,963 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:01:03,963 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:01:03,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:01:03,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:01:03,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51a6b957, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:01:03,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:01:03,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:03,964 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:01:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d34c1f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,965 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b279a63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,965 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:01:03,965 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:01:03,965 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:01:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:01:03,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:01:03,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c024241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:01:03,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:01:03,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:03,967 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:01:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17abac74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10fb506a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,968 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,39449,-1 for getting cluster id 2024-12-04T15:01:03,968 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:01:03,969 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e66e92f8-1d08-41ec-82b1-ea6168d3b6c4' 2024-12-04T15:01:03,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:01:03,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e66e92f8-1d08-41ec-82b1-ea6168d3b6c4" 2024-12-04T15:01:03,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71aa9bd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,39449,-1] 2024-12-04T15:01:03,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:01:03,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:03,970 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:01:03,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a6a1ca9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:01:03,971 DEBUG [Thread-2456 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:01:03,971 DEBUG [Thread-2441 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:01:03,971 DEBUG [Thread-2439 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:01:03,971 DEBUG [Thread-2450 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:01:03,971 DEBUG [Thread-2437 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:01:03,971 DEBUG [Thread-2445 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:01:03,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:01:03,972 DEBUG [Thread-2452 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:01:03,972 DEBUG [Thread-2443 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:01:03,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:01:03,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:01:03,972 DEBUG [Thread-2448 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:01:03,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:01:03,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:01:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:01:03,973 DEBUG [Thread-2454 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:01:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:01:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:01:03,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:01:03,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:01:03,974 DEBUG [Thread-2441 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:01:03,974 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:01:03,974 DEBUG [Thread-2441 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:01:03,974 DEBUG [Thread-2445 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:01:03,974 DEBUG [Thread-2445 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:01:03,974 DEBUG [Thread-2450 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:01:03,974 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:01:03,974 DEBUG [Thread-2450 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:01:03,974 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:01:03,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:01:03,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:01:03,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:01:03,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:01:03,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:01:03,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:01:03,976 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:01:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,36649,1733324304801, seqNum=-1] 2024-12-04T15:01:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:01:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T15:01:03,976 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:01:03,977 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:01:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T15:01:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T15:01:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T15:01:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T15:01:03,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T15:01:03,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T15:01:03,979 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T15:01:03,979 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:01:03,979 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T15:01:03,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-12-04T15:01:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5] 2024-12-04T15:01:03,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T15:01:03,981 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:01:03,982 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:01:03,982 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:01:03,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:03,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9bc7edf6120a83a9451ae3a6357eaeb9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T15:01:03,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=A 2024-12-04T15:01:03,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:03,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=B 2024-12-04T15:01:03,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:03,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=C 2024-12-04T15:01:03,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:04,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120446eaba3856344d4bb8c674ede2c9a7b1_9bc7edf6120a83a9451ae3a6357eaeb9 is 50, key is test_row_0/A:col10/1733324463980/Put/seqid=0 2024-12-04T15:01:04,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742372_1548 (size=12154) 2024-12-04T15:01:04,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324464017, max timestamp=1733324464022Num uniques 859; 2024-12-04T15:01:04,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T15:01:04,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324464020, max timestamp=1733324464022Num uniques 859; 2024-12-04T15:01:04,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T15:01:04,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324464020, max timestamp=1733324464022Num uniques 859; 2024-12-04T15:01:04,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T15:01:04,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:04,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733324524039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:04,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:04,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324524040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:04,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324524040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:04,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:04,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324524040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:04,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:04,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43180 deadline: 1733324524041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:04,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:04,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:04,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T15:01:04,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-04T15:01:04,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:04,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:04,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:04,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:04,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:04,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:04,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:04,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733324524150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:04,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:04,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324524150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:04,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:04,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324524150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324524150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43180 deadline: 1733324524150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:04,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-04T15:01:04,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:04,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:04,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:04,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:04,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T15:01:04,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:04,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733324524360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:04,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:04,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324524360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:04,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:04,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:04,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324524360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:04,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:04,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:04,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:04,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324524361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:04,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:04,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:04,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43180 deadline: 1733324524361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:04,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:04,405 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:04,408 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120446eaba3856344d4bb8c674ede2c9a7b1_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120446eaba3856344d4bb8c674ede2c9a7b1_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:04,409 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/1a9f523feb5f43b29503e27a03b916ff, store: [table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:04,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/1a9f523feb5f43b29503e27a03b916ff is 175, key is test_row_0/A:col10/1733324463980/Put/seqid=0 2024-12-04T15:01:04,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742373_1549 (size=30955) 2024-12-04T15:01:04,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-04T15:01:04,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:04,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:04,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:04,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:04,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:04,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-04T15:01:04,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:04,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:04,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:04,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:04,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T15:01:04,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:04,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324524670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733324524671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:04,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:04,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324524671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324524671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:04,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:04,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43180 deadline: 1733324524671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:04,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-04T15:01:04,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:04,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:04,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:04,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:04,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:04,814 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/1a9f523feb5f43b29503e27a03b916ff 2024-12-04T15:01:04,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/c78c91649e824390a14fd1ad8c23b90e is 50, key is test_row_0/B:col10/1733324463980/Put/seqid=0 2024-12-04T15:01:04,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742374_1550 (size=12001) 2024-12-04T15:01:04,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-04T15:01:04,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:04,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:04,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:04,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:04,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:04,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:05,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-04T15:01:05,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:05,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:05,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:05,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:05,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T15:01:05,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324525180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:05,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733324525180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:05,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43180 deadline: 1733324525180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:05,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324525180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:05,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324525181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:05,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:05,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:05,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:05,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-04T15:01:05,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:05,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:05,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:05,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:05,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:05,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/c78c91649e824390a14fd1ad8c23b90e 2024-12-04T15:01:05,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/eaa4632ab1e14ff9aa75cc41ce21664d is 50, key is test_row_0/C:col10/1733324463980/Put/seqid=0 2024-12-04T15:01:05,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742375_1551 (size=12001) 2024-12-04T15:01:05,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/eaa4632ab1e14ff9aa75cc41ce21664d 2024-12-04T15:01:05,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/1a9f523feb5f43b29503e27a03b916ff as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/1a9f523feb5f43b29503e27a03b916ff 2024-12-04T15:01:05,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/1a9f523feb5f43b29503e27a03b916ff, entries=150, sequenceid=14, filesize=30.2 K 2024-12-04T15:01:05,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/c78c91649e824390a14fd1ad8c23b90e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/c78c91649e824390a14fd1ad8c23b90e 2024-12-04T15:01:05,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/c78c91649e824390a14fd1ad8c23b90e, entries=150, sequenceid=14, filesize=11.7 K 2024-12-04T15:01:05,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/eaa4632ab1e14ff9aa75cc41ce21664d as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/eaa4632ab1e14ff9aa75cc41ce21664d 2024-12-04T15:01:05,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/eaa4632ab1e14ff9aa75cc41ce21664d, entries=150, sequenceid=14, filesize=11.7 K 2024-12-04T15:01:05,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 1306ms, sequenceid=14, compaction requested=false 2024-12-04T15:01:05,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:05,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-04T15:01:05,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:05,361 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 9bc7edf6120a83a9451ae3a6357eaeb9 3/3 column families, dataSize=136.42 KB heapSize=390.06 KB 2024-12-04T15:01:05,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=A 2024-12-04T15:01:05,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:01:05,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=B 2024-12-04T15:01:05,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:01:05,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=C 2024-12-04T15:01:05,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:01:05,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120442f56d6ebbdb4285a6cfaa2187fa3980_9bc7edf6120a83a9451ae3a6357eaeb9 is 50, key is test_row_0/A:col10/1733324464040/Put/seqid=0 2024-12-04T15:01:05,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742376_1552 (size=12154) 2024-12-04T15:01:05,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,798 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120442f56d6ebbdb4285a6cfaa2187fa3980_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120442f56d6ebbdb4285a6cfaa2187fa3980_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:05,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/ae6ba2a183a144f2af3b0738c699b3f7, store: [table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:05,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/ae6ba2a183a144f2af3b0738c699b3f7 is 175, key is test_row_0/A:col10/1733324464040/Put/seqid=0 2024-12-04T15:01:05,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742377_1553 (size=30955) 2024-12-04T15:01:05,804 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/ae6ba2a183a144f2af3b0738c699b3f7 2024-12-04T15:01:05,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/ae407c74e7714c1b9633735926c231e6 is 50, key is test_row_0/B:col10/1733324464040/Put/seqid=0 2024-12-04T15:01:05,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742378_1554 (size=12001) 2024-12-04T15:01:05,816 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/ae407c74e7714c1b9633735926c231e6 2024-12-04T15:01:05,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/532e380a552b48ce8bae40a3f5f4bff5 is 50, key is test_row_0/C:col10/1733324464040/Put/seqid=0 2024-12-04T15:01:05,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742379_1555 (size=12001) 2024-12-04T15:01:05,844 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/532e380a552b48ce8bae40a3f5f4bff5 2024-12-04T15:01:05,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/ae6ba2a183a144f2af3b0738c699b3f7 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/ae6ba2a183a144f2af3b0738c699b3f7 2024-12-04T15:01:05,850 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/ae6ba2a183a144f2af3b0738c699b3f7, entries=150, sequenceid=51, filesize=30.2 K 2024-12-04T15:01:05,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/ae407c74e7714c1b9633735926c231e6 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/ae407c74e7714c1b9633735926c231e6 2024-12-04T15:01:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,856 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/ae407c74e7714c1b9633735926c231e6, entries=150, sequenceid=51, filesize=11.7 K 2024-12-04T15:01:05,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/532e380a552b48ce8bae40a3f5f4bff5 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/532e380a552b48ce8bae40a3f5f4bff5 2024-12-04T15:01:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,859 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/532e380a552b48ce8bae40a3f5f4bff5, entries=150, sequenceid=51, filesize=11.7 K 2024-12-04T15:01:05,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,860 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~390.02 KB/399376, currentSize=0 B/0 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 499ms, sequenceid=51, compaction requested=false 2024-12-04T15:01:05,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:05,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:05,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-04T15:01:05,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-04T15:01:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-04T15:01:05,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8800 sec 2024-12-04T15:01:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.8900 sec 2024-12-04T15:01:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T15:01:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,141 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:01:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,141 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:01:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-04T15:01:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,143 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:01:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T15:01:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,144 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:01:06,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:01:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9bc7edf6120a83a9451ae3a6357eaeb9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T15:01:06,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=A 2024-12-04T15:01:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:06,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=B 2024-12-04T15:01:06,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:06,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=C 2024-12-04T15:01:06,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T15:01:06,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e4f4dc2e4f934e7cae713bd6af8295bf_9bc7edf6120a83a9451ae3a6357eaeb9 is 50, key is test_row_0/A:col10/1733324466226/Put/seqid=0 2024-12-04T15:01:06,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733324466240, max timestamp=1733324466257Num uniques 859; 2024-12-04T15:01:06,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733324466240, max timestamp=1733324466257Num uniques 859; 2024-12-04T15:01:06,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T15:01:06,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T15:01:06,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733324466240, max timestamp=1733324466257Num uniques 859; 2024-12-04T15:01:06,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T15:01:06,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742380_1556 (size=21918) 2024-12-04T15:01:06,272 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:06,276 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e4f4dc2e4f934e7cae713bd6af8295bf_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e4f4dc2e4f934e7cae713bd6af8295bf_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:06,277 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/bcebc65e7d7a4b05aaacdd4a4e44c5c7, store: [table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:06,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/bcebc65e7d7a4b05aaacdd4a4e44c5c7 is 175, key is test_row_0/A:col10/1733324466226/Put/seqid=0 2024-12-04T15:01:06,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-04T15:01:06,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:06,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:06,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:06,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:06,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:06,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742381_1557 (size=65323) 2024-12-04T15:01:06,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324526285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:06,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324526287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:06,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324526313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:06,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733324526313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:06,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43180 deadline: 1733324526317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:06,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324526420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:06,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:06,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324526421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:06,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:06,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324526421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:06,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733324526430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:06,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:06,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43180 deadline: 1733324526430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:06,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:06,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:06,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-04T15:01:06,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:06,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:06,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:06,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:06,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T15:01:06,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-04T15:01:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:06,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:06,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324526630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:06,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324526630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:06,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324526630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:06,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733324526640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43180 deadline: 1733324526640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:06,710 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/bcebc65e7d7a4b05aaacdd4a4e44c5c7 2024-12-04T15:01:06,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/35ac3d6b9940478cab76df6a77142649 is 50, key is test_row_0/B:col10/1733324466226/Put/seqid=0 2024-12-04T15:01:06,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742382_1558 (size=12001) 2024-12-04T15:01:06,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/35ac3d6b9940478cab76df6a77142649 2024-12-04T15:01:06,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/6bf8bebc02a34eeabd4b99c7cfe7bcf1 is 50, key is test_row_0/C:col10/1733324466226/Put/seqid=0 2024-12-04T15:01:06,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742383_1559 (size=12001) 2024-12-04T15:01:06,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-04T15:01:06,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:06,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:06,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:06,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:06,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:06,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T15:01:06,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-04T15:01:06,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:06,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:06,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:06,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:06,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:06,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324526940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:06,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:06,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324526940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:06,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:06,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324526940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:06,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:06,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733324526950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:06,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43180 deadline: 1733324526950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:06,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:06,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:07,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-04T15:01:07,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:07,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:07,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:07,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:07,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:07,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/6bf8bebc02a34eeabd4b99c7cfe7bcf1 2024-12-04T15:01:07,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/bcebc65e7d7a4b05aaacdd4a4e44c5c7 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/bcebc65e7d7a4b05aaacdd4a4e44c5c7 2024-12-04T15:01:07,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/bcebc65e7d7a4b05aaacdd4a4e44c5c7, entries=350, sequenceid=61, filesize=63.8 K 2024-12-04T15:01:07,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/35ac3d6b9940478cab76df6a77142649 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/35ac3d6b9940478cab76df6a77142649 2024-12-04T15:01:07,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/35ac3d6b9940478cab76df6a77142649, entries=150, sequenceid=61, filesize=11.7 K 2024-12-04T15:01:07,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/6bf8bebc02a34eeabd4b99c7cfe7bcf1 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/6bf8bebc02a34eeabd4b99c7cfe7bcf1 2024-12-04T15:01:07,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/6bf8bebc02a34eeabd4b99c7cfe7bcf1, entries=150, sequenceid=61, filesize=11.7 K 2024-12-04T15:01:07,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 935ms, sequenceid=61, compaction requested=true 2024-12-04T15:01:07,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:07,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:01:07,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:07,165 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:07,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:01:07,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:07,165 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:07,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:01:07,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:01:07,166 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:07,166 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/B is initiating minor compaction (all files) 2024-12-04T15:01:07,166 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/B in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:07,166 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/c78c91649e824390a14fd1ad8c23b90e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/ae407c74e7714c1b9633735926c231e6, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/35ac3d6b9940478cab76df6a77142649] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=35.2 K 2024-12-04T15:01:07,166 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:07,166 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/A is initiating minor compaction (all files) 2024-12-04T15:01:07,167 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/A in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:07,167 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/1a9f523feb5f43b29503e27a03b916ff, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/ae6ba2a183a144f2af3b0738c699b3f7, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/bcebc65e7d7a4b05aaacdd4a4e44c5c7] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=124.3 K 2024-12-04T15:01:07,167 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:07,167 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/1a9f523feb5f43b29503e27a03b916ff, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/ae6ba2a183a144f2af3b0738c699b3f7, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/bcebc65e7d7a4b05aaacdd4a4e44c5c7] 2024-12-04T15:01:07,167 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting c78c91649e824390a14fd1ad8c23b90e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733324463980 2024-12-04T15:01:07,167 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a9f523feb5f43b29503e27a03b916ff, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733324463980 2024-12-04T15:01:07,167 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae6ba2a183a144f2af3b0738c699b3f7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733324464038 2024-12-04T15:01:07,168 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting ae407c74e7714c1b9633735926c231e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733324464038 2024-12-04T15:01:07,168 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 35ac3d6b9940478cab76df6a77142649, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733324466218 2024-12-04T15:01:07,168 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcebc65e7d7a4b05aaacdd4a4e44c5c7, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733324466216 2024-12-04T15:01:07,173 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:07,175 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120410e9ea96ef854d9b813b6b61b00c77cf_9bc7edf6120a83a9451ae3a6357eaeb9 store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:07,175 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#B#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:07,175 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/f372fbc0b0c3411fb1cfcd9f8d20e5b3 is 50, key is test_row_0/B:col10/1733324466226/Put/seqid=0 2024-12-04T15:01:07,177 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120410e9ea96ef854d9b813b6b61b00c77cf_9bc7edf6120a83a9451ae3a6357eaeb9, store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:07,177 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120410e9ea96ef854d9b813b6b61b00c77cf_9bc7edf6120a83a9451ae3a6357eaeb9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:07,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742384_1560 (size=12104) 2024-12-04T15:01:07,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742385_1561 (size=4469) 2024-12-04T15:01:07,199 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T15:01:07,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-04T15:01:07,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:07,211 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 9bc7edf6120a83a9451ae3a6357eaeb9 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-04T15:01:07,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=A 2024-12-04T15:01:07,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:01:07,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=B 2024-12-04T15:01:07,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:01:07,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=C 2024-12-04T15:01:07,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:01:07,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c7b20f55ece54d4685c6a2f0db8f7c81_9bc7edf6120a83a9451ae3a6357eaeb9 is 50, key is test_row_0/A:col10/1733324466286/Put/seqid=0 2024-12-04T15:01:07,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742386_1562 (size=12154) 2024-12-04T15:01:07,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:07,263 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c7b20f55ece54d4685c6a2f0db8f7c81_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c7b20f55ece54d4685c6a2f0db8f7c81_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:07,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/f67adf1c36794312b504fbf77365c58a, store: [table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:07,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/f67adf1c36794312b504fbf77365c58a is 175, key is test_row_0/A:col10/1733324466286/Put/seqid=0 2024-12-04T15:01:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T15:01:07,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742387_1563 (size=30955) 2024-12-04T15:01:07,285 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/f67adf1c36794312b504fbf77365c58a 2024-12-04T15:01:07,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/528160467848429a8a9151ec6607b25d is 50, key is test_row_0/B:col10/1733324466286/Put/seqid=0 2024-12-04T15:01:07,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742388_1564 (size=12001) 2024-12-04T15:01:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:07,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:07,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324527459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324527460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:07,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733324527460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43180 deadline: 1733324527460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:07,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324527464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:07,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:07,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324527570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:07,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:07,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324527570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:07,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:07,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:07,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:07,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324527581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:07,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:07,585 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/f372fbc0b0c3411fb1cfcd9f8d20e5b3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/f372fbc0b0c3411fb1cfcd9f8d20e5b3 2024-12-04T15:01:07,590 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/B of 9bc7edf6120a83a9451ae3a6357eaeb9 into f372fbc0b0c3411fb1cfcd9f8d20e5b3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:07,590 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:07,590 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/B, priority=13, startTime=1733324467165; duration=0sec 2024-12-04T15:01:07,590 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:01:07,590 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:B 2024-12-04T15:01:07,590 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:07,591 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:07,591 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/C is initiating minor compaction (all files) 2024-12-04T15:01:07,591 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/C in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:07,591 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/eaa4632ab1e14ff9aa75cc41ce21664d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/532e380a552b48ce8bae40a3f5f4bff5, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/6bf8bebc02a34eeabd4b99c7cfe7bcf1] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=35.2 K 2024-12-04T15:01:07,603 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#A#compaction#475 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:07,603 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting eaa4632ab1e14ff9aa75cc41ce21664d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733324463980 2024-12-04T15:01:07,603 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/d8710a2b94e346a5a7198b4703ea849e is 175, key is test_row_0/A:col10/1733324466226/Put/seqid=0 2024-12-04T15:01:07,603 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 532e380a552b48ce8bae40a3f5f4bff5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733324464038 2024-12-04T15:01:07,604 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bf8bebc02a34eeabd4b99c7cfe7bcf1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733324466218 2024-12-04T15:01:07,620 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#C#compaction#478 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:07,620 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/1e4ef57e37ad4fd5a340a9d15194b09e is 50, key is test_row_0/C:col10/1733324466226/Put/seqid=0 2024-12-04T15:01:07,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742389_1565 (size=31058) 2024-12-04T15:01:07,630 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/d8710a2b94e346a5a7198b4703ea849e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d8710a2b94e346a5a7198b4703ea849e 2024-12-04T15:01:07,638 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/A of 9bc7edf6120a83a9451ae3a6357eaeb9 into d8710a2b94e346a5a7198b4703ea849e(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:07,638 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:07,638 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/A, priority=13, startTime=1733324467165; duration=0sec 2024-12-04T15:01:07,638 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:07,638 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:A 2024-12-04T15:01:07,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742390_1566 (size=12104) 2024-12-04T15:01:07,672 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/1e4ef57e37ad4fd5a340a9d15194b09e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/1e4ef57e37ad4fd5a340a9d15194b09e 2024-12-04T15:01:07,675 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/C of 9bc7edf6120a83a9451ae3a6357eaeb9 into 1e4ef57e37ad4fd5a340a9d15194b09e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:07,675 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:07,675 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/C, priority=13, startTime=1733324467165; duration=0sec 2024-12-04T15:01:07,675 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:07,676 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:C 2024-12-04T15:01:07,697 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/528160467848429a8a9151ec6607b25d 2024-12-04T15:01:07,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/ee5ad04e625b4397ae3423cb0e0bb92a is 50, key is test_row_0/C:col10/1733324466286/Put/seqid=0 2024-12-04T15:01:07,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742391_1567 (size=12001) 2024-12-04T15:01:07,705 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/ee5ad04e625b4397ae3423cb0e0bb92a 2024-12-04T15:01:07,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/f67adf1c36794312b504fbf77365c58a as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/f67adf1c36794312b504fbf77365c58a 2024-12-04T15:01:07,711 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/f67adf1c36794312b504fbf77365c58a, entries=150, sequenceid=97, filesize=30.2 K 2024-12-04T15:01:07,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/528160467848429a8a9151ec6607b25d as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/528160467848429a8a9151ec6607b25d 2024-12-04T15:01:07,714 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/528160467848429a8a9151ec6607b25d, entries=150, sequenceid=97, filesize=11.7 K 2024-12-04T15:01:07,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/ee5ad04e625b4397ae3423cb0e0bb92a as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/ee5ad04e625b4397ae3423cb0e0bb92a 2024-12-04T15:01:07,717 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/ee5ad04e625b4397ae3423cb0e0bb92a, entries=150, sequenceid=97, filesize=11.7 K 2024-12-04T15:01:07,718 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 507ms, sequenceid=97, compaction requested=false 2024-12-04T15:01:07,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:07,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:07,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-04T15:01:07,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-04T15:01:07,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-04T15:01:07,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5750 sec 2024-12-04T15:01:07,722 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.5800 sec 2024-12-04T15:01:07,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:07,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9bc7edf6120a83a9451ae3a6357eaeb9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T15:01:07,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=A 2024-12-04T15:01:07,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:07,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=B 2024-12-04T15:01:07,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:07,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=C 2024-12-04T15:01:07,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:07,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c7f1c968286e49aaa352dbe61bf4dec6_9bc7edf6120a83a9451ae3a6357eaeb9 is 50, key is test_row_0/A:col10/1733324467781/Put/seqid=0 2024-12-04T15:01:07,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742392_1568 (size=17034) 2024-12-04T15:01:07,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324467801, max timestamp=1733324467814Num uniques 859; 2024-12-04T15:01:07,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T15:01:07,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324467801, max timestamp=1733324467814Num uniques 859; 2024-12-04T15:01:07,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T15:01:07,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733324467801, max timestamp=1733324467814Num uniques 859; 2024-12-04T15:01:07,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-04T15:01:07,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:07,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324527837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:07,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:07,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324527843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:07,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:07,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324527843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:07,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:07,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324527950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:07,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:07,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324527950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:07,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:07,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:07,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:07,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324527960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:08,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:08,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324528160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:08,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:08,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324528160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:08,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:08,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:08,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:08,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324528170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:08,193 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:08,197 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c7f1c968286e49aaa352dbe61bf4dec6_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c7f1c968286e49aaa352dbe61bf4dec6_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:08,198 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/a9ecfd1e5b0845cd9097c954fdaca840, store: [table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:08,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/a9ecfd1e5b0845cd9097c954fdaca840 is 175, key is test_row_0/A:col10/1733324467781/Put/seqid=0 2024-12-04T15:01:08,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742393_1569 (size=48139) 2024-12-04T15:01:08,202 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=111, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/a9ecfd1e5b0845cd9097c954fdaca840 2024-12-04T15:01:08,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/740b364c50cd4cff8d1de379beec8515 is 50, key is test_row_0/B:col10/1733324467781/Put/seqid=0 2024-12-04T15:01:08,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742394_1570 (size=12001) 2024-12-04T15:01:08,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T15:01:08,290 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:01:08,291 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:01:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-04T15:01:08,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T15:01:08,292 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:01:08,293 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:01:08,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:01:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T15:01:08,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-04T15:01:08,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:08,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:08,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:08,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:08,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:08,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:08,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324528470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:08,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:08,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324528470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:08,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:08,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:08,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:08,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733324528480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:08,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:08,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43180 deadline: 1733324528480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:08,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:08,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324528480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:08,484 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 9bc7edf6120a83a9451ae3a6357eaeb9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:01:08,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:08,485 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 9bc7edf6120a83a9451ae3a6357eaeb9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:01:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:08,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-04T15:01:08,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:08,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:08,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:08,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:08,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:08,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:08,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T15:01:08,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/740b364c50cd4cff8d1de379beec8515 2024-12-04T15:01:08,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/e1521509de1845e9953c922772f47480 is 50, key is test_row_0/C:col10/1733324467781/Put/seqid=0 2024-12-04T15:01:08,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742395_1571 (size=12001) 2024-12-04T15:01:08,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-04T15:01:08,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:08,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:08,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:08,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:08,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:08,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:08,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-04T15:01:08,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:08,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:08,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:08,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:08,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T15:01:08,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324528980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324528980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:08,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324528990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:08,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:08,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:09,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/e1521509de1845e9953c922772f47480 2024-12-04T15:01:09,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/a9ecfd1e5b0845cd9097c954fdaca840 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/a9ecfd1e5b0845cd9097c954fdaca840 2024-12-04T15:01:09,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/a9ecfd1e5b0845cd9097c954fdaca840, entries=250, sequenceid=111, filesize=47.0 K 2024-12-04T15:01:09,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/740b364c50cd4cff8d1de379beec8515 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/740b364c50cd4cff8d1de379beec8515 2024-12-04T15:01:09,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/740b364c50cd4cff8d1de379beec8515, entries=150, sequenceid=111, filesize=11.7 K 2024-12-04T15:01:09,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/e1521509de1845e9953c922772f47480 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/e1521509de1845e9953c922772f47480 2024-12-04T15:01:09,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/e1521509de1845e9953c922772f47480, entries=150, sequenceid=111, filesize=11.7 K 2024-12-04T15:01:09,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.55 KB/134706 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 1268ms, sequenceid=111, compaction requested=true 2024-12-04T15:01:09,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:09,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:01:09,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:09,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:01:09,050 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:09,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:09,050 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:09,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:01:09,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:01:09,050 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:09,050 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:09,050 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/A is initiating minor compaction (all files) 2024-12-04T15:01:09,050 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/B is initiating minor compaction (all files) 2024-12-04T15:01:09,050 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/A in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:09,050 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d8710a2b94e346a5a7198b4703ea849e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/f67adf1c36794312b504fbf77365c58a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/a9ecfd1e5b0845cd9097c954fdaca840] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=107.6 K 2024-12-04T15:01:09,050 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:09,050 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d8710a2b94e346a5a7198b4703ea849e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/f67adf1c36794312b504fbf77365c58a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/a9ecfd1e5b0845cd9097c954fdaca840] 2024-12-04T15:01:09,050 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/B in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:09,050 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/f372fbc0b0c3411fb1cfcd9f8d20e5b3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/528160467848429a8a9151ec6607b25d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/740b364c50cd4cff8d1de379beec8515] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=35.3 K 2024-12-04T15:01:09,051 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8710a2b94e346a5a7198b4703ea849e, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733324466218 2024-12-04T15:01:09,051 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f67adf1c36794312b504fbf77365c58a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733324466285 2024-12-04T15:01:09,051 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9ecfd1e5b0845cd9097c954fdaca840, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733324467457 2024-12-04T15:01:09,051 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting f372fbc0b0c3411fb1cfcd9f8d20e5b3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733324466218 2024-12-04T15:01:09,051 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 528160467848429a8a9151ec6607b25d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733324466285 2024-12-04T15:01:09,052 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 740b364c50cd4cff8d1de379beec8515, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733324467462 2024-12-04T15:01:09,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-04T15:01:09,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:09,055 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 9bc7edf6120a83a9451ae3a6357eaeb9 3/3 column families, dataSize=131.55 KB heapSize=374.30 KB 2024-12-04T15:01:09,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=A 2024-12-04T15:01:09,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:01:09,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=B 2024-12-04T15:01:09,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:01:09,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=C 2024-12-04T15:01:09,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:01:09,056 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:09,057 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#B#compaction#484 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:09,057 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/d3a7bdd75df44aac83663fc6a5896483 is 50, key is test_row_0/B:col10/1733324467781/Put/seqid=0 2024-12-04T15:01:09,063 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412049d4b8924bdf5483a96238d81febcc8ea_9bc7edf6120a83a9451ae3a6357eaeb9 store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:09,064 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412049d4b8924bdf5483a96238d81febcc8ea_9bc7edf6120a83a9451ae3a6357eaeb9, store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:09,065 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412049d4b8924bdf5483a96238d81febcc8ea_9bc7edf6120a83a9451ae3a6357eaeb9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:09,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742396_1572 (size=12207) 2024-12-04T15:01:09,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120422e489b6e37140fba6b4cbeb29cb5f1c_9bc7edf6120a83a9451ae3a6357eaeb9 is 50, key is test_row_0/A:col10/1733324467841/Put/seqid=0 2024-12-04T15:01:09,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742397_1573 (size=4469) 2024-12-04T15:01:09,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742398_1574 (size=12304) 2024-12-04T15:01:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T15:01:09,474 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/d3a7bdd75df44aac83663fc6a5896483 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/d3a7bdd75df44aac83663fc6a5896483 2024-12-04T15:01:09,477 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/B of 9bc7edf6120a83a9451ae3a6357eaeb9 into d3a7bdd75df44aac83663fc6a5896483(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:09,477 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:09,477 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/B, priority=13, startTime=1733324469050; duration=0sec 2024-12-04T15:01:09,477 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:01:09,477 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:B 2024-12-04T15:01:09,478 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:09,478 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:09,478 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/C is initiating minor compaction (all files) 2024-12-04T15:01:09,478 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/C in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:09,478 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/1e4ef57e37ad4fd5a340a9d15194b09e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/ee5ad04e625b4397ae3423cb0e0bb92a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/e1521509de1845e9953c922772f47480] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=35.3 K 2024-12-04T15:01:09,479 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e4ef57e37ad4fd5a340a9d15194b09e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733324466218 2024-12-04T15:01:09,479 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting ee5ad04e625b4397ae3423cb0e0bb92a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733324466285 2024-12-04T15:01:09,479 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting e1521509de1845e9953c922772f47480, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733324467462 2024-12-04T15:01:09,483 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#C#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:09,483 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/191db78b85e84a5ba8bf595506309c36 is 50, key is test_row_0/C:col10/1733324467781/Put/seqid=0 2024-12-04T15:01:09,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742399_1575 (size=12207) 2024-12-04T15:01:09,493 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#A#compaction#483 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:09,494 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/01d6642efb1049ab97c0ae728318364b is 175, key is test_row_0/A:col10/1733324467781/Put/seqid=0 2024-12-04T15:01:09,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742400_1576 (size=31161) 2024-12-04T15:01:09,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:09,509 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120422e489b6e37140fba6b4cbeb29cb5f1c_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120422e489b6e37140fba6b4cbeb29cb5f1c_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:09,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/0d88fc2dcb084d8483d854adbc29b6ae, store: [table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:09,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/0d88fc2dcb084d8483d854adbc29b6ae is 175, key is test_row_0/A:col10/1733324467841/Put/seqid=0 2024-12-04T15:01:09,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742401_1577 (size=31105) 2024-12-04T15:01:09,525 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/0d88fc2dcb084d8483d854adbc29b6ae 2024-12-04T15:01:09,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/b47990a598644af0a22871def6575f1b is 50, key is test_row_0/B:col10/1733324467841/Put/seqid=0 2024-12-04T15:01:09,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742402_1578 (size=12151) 2024-12-04T15:01:09,894 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/191db78b85e84a5ba8bf595506309c36 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/191db78b85e84a5ba8bf595506309c36 2024-12-04T15:01:09,904 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/01d6642efb1049ab97c0ae728318364b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/01d6642efb1049ab97c0ae728318364b 2024-12-04T15:01:09,907 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/C of 9bc7edf6120a83a9451ae3a6357eaeb9 into 191db78b85e84a5ba8bf595506309c36(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:09,907 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:09,907 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/C, priority=13, startTime=1733324469050; duration=0sec 2024-12-04T15:01:09,907 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:09,907 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:C 2024-12-04T15:01:09,910 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/A of 9bc7edf6120a83a9451ae3a6357eaeb9 into 01d6642efb1049ab97c0ae728318364b(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:09,910 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:09,910 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/A, priority=13, startTime=1733324469049; duration=0sec 2024-12-04T15:01:09,910 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:09,910 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:A 2024-12-04T15:01:09,937 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/b47990a598644af0a22871def6575f1b 2024-12-04T15:01:09,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/db4ecf1c36ac4b70b1879f5579849dcb is 50, key is test_row_0/C:col10/1733324467841/Put/seqid=0 2024-12-04T15:01:09,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742403_1579 (size=12151) 2024-12-04T15:01:09,969 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.10 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/db4ecf1c36ac4b70b1879f5579849dcb 2024-12-04T15:01:09,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/0d88fc2dcb084d8483d854adbc29b6ae as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/0d88fc2dcb084d8483d854adbc29b6ae 2024-12-04T15:01:09,987 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/0d88fc2dcb084d8483d854adbc29b6ae, entries=150, sequenceid=147, filesize=30.4 K 2024-12-04T15:01:09,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/b47990a598644af0a22871def6575f1b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/b47990a598644af0a22871def6575f1b 2024-12-04T15:01:09,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:09,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:09,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:09,995 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/b47990a598644af0a22871def6575f1b, entries=150, sequenceid=147, filesize=11.9 K 2024-12-04T15:01:09,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/db4ecf1c36ac4b70b1879f5579849dcb as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/db4ecf1c36ac4b70b1879f5579849dcb 2024-12-04T15:01:09,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:09,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:09,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:09,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:09,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,001 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/db4ecf1c36ac4b70b1879f5579849dcb, entries=150, sequenceid=147, filesize=11.9 K 2024-12-04T15:01:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,002 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~131.55 KB/134706, heapSize ~374.26 KB/383240, currentSize=0 B/0 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 947ms, sequenceid=147, compaction requested=false 2024-12-04T15:01:10,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:10,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:10,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-04T15:01:10,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-04T15:01:10,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-04T15:01:10,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7110 sec 2024-12-04T15:01:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.7160 sec 2024-12-04T15:01:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:10,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9bc7edf6120a83a9451ae3a6357eaeb9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T15:01:10,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=A 2024-12-04T15:01:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:10,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=B 2024-12-04T15:01:10,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:10,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=C 2024-12-04T15:01:10,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412048662564f24b04c3f884d4b8f804ffd10_9bc7edf6120a83a9451ae3a6357eaeb9 is 50, key is test_row_0/A:col10/1733324470082/Put/seqid=0 2024-12-04T15:01:10,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742404_1580 (size=17284) 2024-12-04T15:01:10,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733324470100, max timestamp=1733324470120Num uniques 859; 2024-12-04T15:01:10,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-04T15:01:10,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733324470100, max timestamp=1733324470120Num uniques 859; 2024-12-04T15:01:10,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-04T15:01:10,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733324470100, max timestamp=1733324470120Num uniques 859; 2024-12-04T15:01:10,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d214edae7bd2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T15:01:10,127 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:10,134 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412048662564f24b04c3f884d4b8f804ffd10_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048662564f24b04c3f884d4b8f804ffd10_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:10,135 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/242a98046a13415b8cbde9af10ae5db2, store: [table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:10,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/242a98046a13415b8cbde9af10ae5db2 is 175, key is test_row_0/A:col10/1733324470082/Put/seqid=0 2024-12-04T15:01:10,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742405_1581 (size=48385) 2024-12-04T15:01:10,142 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/242a98046a13415b8cbde9af10ae5db2 2024-12-04T15:01:10,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/92b3c753f5b6404f86bceff1815263db is 50, key is test_row_0/B:col10/1733324470082/Put/seqid=0 2024-12-04T15:01:10,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742406_1582 (size=9757) 2024-12-04T15:01:10,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/92b3c753f5b6404f86bceff1815263db 2024-12-04T15:01:10,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/f127d2edfb144927bfac25a031b0ebf3 is 50, key is test_row_0/C:col10/1733324470082/Put/seqid=0 2024-12-04T15:01:10,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742407_1583 (size=9757) 2024-12-04T15:01:10,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:10,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324530165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:10,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:10,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:10,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324530165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324530166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:10,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324530280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324530281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:10,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324530281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T15:01:10,440 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:01:10,441 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:01:10,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-04T15:01:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T15:01:10,442 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:01:10,443 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:01:10,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:01:10,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324530490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:10,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324530490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:10,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324530490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:10,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43180 deadline: 1733324530510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:10,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733324530510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:10,512 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 9bc7edf6120a83a9451ae3a6357eaeb9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:01:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:10,512 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 9bc7edf6120a83a9451ae3a6357eaeb9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:01:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T15:01:10,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/f127d2edfb144927bfac25a031b0ebf3 2024-12-04T15:01:10,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/242a98046a13415b8cbde9af10ae5db2 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/242a98046a13415b8cbde9af10ae5db2 2024-12-04T15:01:10,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/242a98046a13415b8cbde9af10ae5db2, entries=250, sequenceid=160, filesize=47.3 K 2024-12-04T15:01:10,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/92b3c753f5b6404f86bceff1815263db as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/92b3c753f5b6404f86bceff1815263db 2024-12-04T15:01:10,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/92b3c753f5b6404f86bceff1815263db, entries=100, sequenceid=160, filesize=9.5 K 2024-12-04T15:01:10,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/f127d2edfb144927bfac25a031b0ebf3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/f127d2edfb144927bfac25a031b0ebf3 2024-12-04T15:01:10,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/f127d2edfb144927bfac25a031b0ebf3, entries=100, sequenceid=160, filesize=9.5 K 2024-12-04T15:01:10,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=135.63 KB/138880 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 497ms, sequenceid=160, compaction requested=true 2024-12-04T15:01:10,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:10,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:01:10,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:10,581 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:10,581 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:10,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:01:10,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:10,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:01:10,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:01:10,582 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:10,582 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:10,582 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/B is initiating minor compaction (all files) 2024-12-04T15:01:10,582 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/A is initiating minor compaction (all files) 2024-12-04T15:01:10,582 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/B in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:10,582 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/A in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:10,582 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/01d6642efb1049ab97c0ae728318364b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/0d88fc2dcb084d8483d854adbc29b6ae, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/242a98046a13415b8cbde9af10ae5db2] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=108.1 K 2024-12-04T15:01:10,582 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/d3a7bdd75df44aac83663fc6a5896483, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/b47990a598644af0a22871def6575f1b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/92b3c753f5b6404f86bceff1815263db] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=33.3 K 2024-12-04T15:01:10,582 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:10,583 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/01d6642efb1049ab97c0ae728318364b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/0d88fc2dcb084d8483d854adbc29b6ae, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/242a98046a13415b8cbde9af10ae5db2] 2024-12-04T15:01:10,583 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting d3a7bdd75df44aac83663fc6a5896483, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733324467462 2024-12-04T15:01:10,583 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01d6642efb1049ab97c0ae728318364b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733324467462 2024-12-04T15:01:10,583 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d88fc2dcb084d8483d854adbc29b6ae, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733324467828 2024-12-04T15:01:10,583 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting b47990a598644af0a22871def6575f1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733324467828 2024-12-04T15:01:10,583 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 242a98046a13415b8cbde9af10ae5db2, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733324470045 2024-12-04T15:01:10,583 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 92b3c753f5b6404f86bceff1815263db, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733324470054 2024-12-04T15:01:10,594 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#B#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:10,595 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/835107bab2f84af8ac5bc368936de601 is 50, key is test_row_0/B:col10/1733324470082/Put/seqid=0 2024-12-04T15:01:10,596 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:10,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-04T15:01:10,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:10,597 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 9bc7edf6120a83a9451ae3a6357eaeb9 3/3 column families, dataSize=135.63 KB heapSize=388.23 KB 2024-12-04T15:01:10,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=A 2024-12-04T15:01:10,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:01:10,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=B 2024-12-04T15:01:10,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:01:10,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=C 2024-12-04T15:01:10,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T15:01:10,601 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204003cd70176a443e292abf07f3a0b2bde_9bc7edf6120a83a9451ae3a6357eaeb9 store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:10,603 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204003cd70176a443e292abf07f3a0b2bde_9bc7edf6120a83a9451ae3a6357eaeb9, store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:10,603 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204003cd70176a443e292abf07f3a0b2bde_9bc7edf6120a83a9451ae3a6357eaeb9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:10,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742408_1584 (size=12459) 2024-12-04T15:01:10,608 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/835107bab2f84af8ac5bc368936de601 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/835107bab2f84af8ac5bc368936de601 2024-12-04T15:01:10,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412049d285a92481a48bebd160fe76932df61_9bc7edf6120a83a9451ae3a6357eaeb9 is 50, key is test_row_0/A:col10/1733324470162/Put/seqid=0 2024-12-04T15:01:10,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742409_1585 (size=4469) 2024-12-04T15:01:10,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742410_1586 (size=12304) 2024-12-04T15:01:10,616 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/B of 9bc7edf6120a83a9451ae3a6357eaeb9 into 835107bab2f84af8ac5bc368936de601(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:10,616 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:10,616 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/B, priority=13, startTime=1733324470581; duration=0sec 2024-12-04T15:01:10,616 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:01:10,616 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:B 2024-12-04T15:01:10,616 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:10,617 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:10,617 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/C is initiating minor compaction (all files) 2024-12-04T15:01:10,617 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/C in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:10,617 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/191db78b85e84a5ba8bf595506309c36, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/db4ecf1c36ac4b70b1879f5579849dcb, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/f127d2edfb144927bfac25a031b0ebf3] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=33.3 K 2024-12-04T15:01:10,617 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 191db78b85e84a5ba8bf595506309c36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733324467462 2024-12-04T15:01:10,617 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting db4ecf1c36ac4b70b1879f5579849dcb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733324467828 2024-12-04T15:01:10,618 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting f127d2edfb144927bfac25a031b0ebf3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733324470054 2024-12-04T15:01:10,621 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#C#compaction#495 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:10,622 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/c6a2d669e1ed4923b413171488bfc45e is 50, key is test_row_0/C:col10/1733324470082/Put/seqid=0 2024-12-04T15:01:10,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742411_1587 (size=12459) 2024-12-04T15:01:10,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T15:01:10,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:10,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:10,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324530811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:10,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324530816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:10,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324530816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:10,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:10,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324530920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:10,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:10,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324530930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:10,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:10,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324530930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:10,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:11,012 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#A#compaction#493 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:11,012 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/203490922aca487dbf0e3f563f770e98 is 175, key is test_row_0/A:col10/1733324470082/Put/seqid=0 2024-12-04T15:01:11,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742412_1588 (size=31520) 2024-12-04T15:01:11,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:11,018 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/203490922aca487dbf0e3f563f770e98 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/203490922aca487dbf0e3f563f770e98 2024-12-04T15:01:11,019 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412049d285a92481a48bebd160fe76932df61_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412049d285a92481a48bebd160fe76932df61_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:11,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/da89b10e39904455850ba781940da40d, store: [table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:11,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/da89b10e39904455850ba781940da40d is 175, key is test_row_0/A:col10/1733324470162/Put/seqid=0 2024-12-04T15:01:11,023 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/A of 9bc7edf6120a83a9451ae3a6357eaeb9 into 203490922aca487dbf0e3f563f770e98(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:11,023 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:11,023 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/A, priority=13, startTime=1733324470581; duration=0sec 2024-12-04T15:01:11,023 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:11,023 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:A 2024-12-04T15:01:11,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742413_1589 (size=31105) 2024-12-04T15:01:11,025 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=44.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/da89b10e39904455850ba781940da40d 2024-12-04T15:01:11,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/79b12ab3ab33468abea445e81007b833 is 50, key is test_row_0/B:col10/1733324470162/Put/seqid=0 2024-12-04T15:01:11,038 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/c6a2d669e1ed4923b413171488bfc45e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/c6a2d669e1ed4923b413171488bfc45e 2024-12-04T15:01:11,043 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/C of 9bc7edf6120a83a9451ae3a6357eaeb9 into c6a2d669e1ed4923b413171488bfc45e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:11,043 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:11,043 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/C, priority=13, startTime=1733324470582; duration=0sec 2024-12-04T15:01:11,043 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:11,043 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:C 2024-12-04T15:01:11,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742414_1590 (size=12151) 2024-12-04T15:01:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T15:01:11,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:11,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324531129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:11,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:11,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:11,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:11,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324531140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:11,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324531140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:11,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:11,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:11,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:11,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324531440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:11,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:11,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:11,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:11,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324531450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:11,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324531450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:11,464 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/79b12ab3ab33468abea445e81007b833 2024-12-04T15:01:11,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/19c1f1a89a9c47fa8f4f41f038d54cea is 50, key is test_row_0/C:col10/1733324470162/Put/seqid=0 2024-12-04T15:01:11,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742415_1591 (size=12151) 2024-12-04T15:01:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T15:01:11,873 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/19c1f1a89a9c47fa8f4f41f038d54cea 2024-12-04T15:01:11,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/da89b10e39904455850ba781940da40d as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/da89b10e39904455850ba781940da40d 2024-12-04T15:01:11,882 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/da89b10e39904455850ba781940da40d, entries=150, sequenceid=197, filesize=30.4 K 2024-12-04T15:01:11,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/79b12ab3ab33468abea445e81007b833 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/79b12ab3ab33468abea445e81007b833 2024-12-04T15:01:11,888 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/79b12ab3ab33468abea445e81007b833, entries=150, sequenceid=197, filesize=11.9 K 2024-12-04T15:01:11,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/19c1f1a89a9c47fa8f4f41f038d54cea as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/19c1f1a89a9c47fa8f4f41f038d54cea 2024-12-04T15:01:11,893 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/19c1f1a89a9c47fa8f4f41f038d54cea, entries=150, sequenceid=197, filesize=11.9 K 2024-12-04T15:01:11,894 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~135.63 KB/138880, heapSize ~388.19 KB/397504, currentSize=53.67 KB/54960 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 1297ms, sequenceid=197, compaction requested=false 2024-12-04T15:01:11,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:11,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:11,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-04T15:01:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-04T15:01:11,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-04T15:01:11,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4520 sec 2024-12-04T15:01:11,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.4560 sec 2024-12-04T15:01:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:11,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9bc7edf6120a83a9451ae3a6357eaeb9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T15:01:11,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=A 2024-12-04T15:01:11,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:11,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=B 2024-12-04T15:01:11,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:11,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=C 2024-12-04T15:01:11,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:11,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120439e711b3909140868f3ddcd5de5c8076_9bc7edf6120a83a9451ae3a6357eaeb9 is 50, key is test_row_0/A:col10/1733324471951/Put/seqid=0 2024-12-04T15:01:11,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742416_1592 (size=14794) 2024-12-04T15:01:11,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324531987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:11,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324531988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:11,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:11,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:11,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:11,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324531990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:12,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324532100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:12,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324532100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:12,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:12,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:12,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:12,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:12,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:12,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:12,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324532100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:12,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:12,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:12,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:12,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:12,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324532311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:12,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:12,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:12,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:12,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324532311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:12,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:12,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:12,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:12,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324532311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:12,381 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:12,394 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120439e711b3909140868f3ddcd5de5c8076_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120439e711b3909140868f3ddcd5de5c8076_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:12,395 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/30c67df7db814eadaa62184c9ebdce22, store: [table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:12,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/30c67df7db814eadaa62184c9ebdce22 is 175, key is test_row_0/A:col10/1733324471951/Put/seqid=0 2024-12-04T15:01:12,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742417_1593 (size=39749) 2024-12-04T15:01:12,426 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/30c67df7db814eadaa62184c9ebdce22 2024-12-04T15:01:12,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/d4bc9fa710074e61bca64acf7ff29050 is 50, key is test_row_0/B:col10/1733324471951/Put/seqid=0 2024-12-04T15:01:12,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742418_1594 (size=12151) 2024-12-04T15:01:12,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/d4bc9fa710074e61bca64acf7ff29050 2024-12-04T15:01:12,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/5b1c17c65b9549c683fdb468ff72c7ce is 50, key is test_row_0/C:col10/1733324471951/Put/seqid=0 2024-12-04T15:01:12,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742419_1595 (size=12151) 2024-12-04T15:01:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T15:01:12,590 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:01:12,591 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:01:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-04T15:01:12,592 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:01:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T15:01:12,593 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:01:12,593 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:01:12,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324532620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:12,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:12,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:12,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:12,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324532620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:12,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324532620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:12,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T15:01:12,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-04T15:01:12,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:12,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:12,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:12,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:12,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:12,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:12,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-04T15:01:12,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:12,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:12,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:12,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:12,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:12,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:12,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T15:01:12,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/5b1c17c65b9549c683fdb468ff72c7ce 2024-12-04T15:01:12,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/30c67df7db814eadaa62184c9ebdce22 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/30c67df7db814eadaa62184c9ebdce22 2024-12-04T15:01:12,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/30c67df7db814eadaa62184c9ebdce22, entries=200, sequenceid=212, filesize=38.8 K 2024-12-04T15:01:12,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/d4bc9fa710074e61bca64acf7ff29050 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/d4bc9fa710074e61bca64acf7ff29050 2024-12-04T15:01:12,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/d4bc9fa710074e61bca64acf7ff29050, entries=150, sequenceid=212, filesize=11.9 K 2024-12-04T15:01:12,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/5b1c17c65b9549c683fdb468ff72c7ce as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/5b1c17c65b9549c683fdb468ff72c7ce 2024-12-04T15:01:12,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/5b1c17c65b9549c683fdb468ff72c7ce, entries=150, sequenceid=212, filesize=11.9 K 2024-12-04T15:01:12,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 1015ms, sequenceid=212, compaction requested=true 2024-12-04T15:01:12,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:12,967 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:12,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:01:12,968 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102374 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:12,968 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/A is initiating minor compaction (all files) 2024-12-04T15:01:12,968 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/A in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:12,968 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/203490922aca487dbf0e3f563f770e98, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/da89b10e39904455850ba781940da40d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/30c67df7db814eadaa62184c9ebdce22] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=100.0 K 2024-12-04T15:01:12,968 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:12,968 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/203490922aca487dbf0e3f563f770e98, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/da89b10e39904455850ba781940da40d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/30c67df7db814eadaa62184c9ebdce22] 2024-12-04T15:01:12,969 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 203490922aca487dbf0e3f563f770e98, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733324467828 2024-12-04T15:01:12,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:12,970 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting da89b10e39904455850ba781940da40d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733324470154 2024-12-04T15:01:12,970 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:12,970 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30c67df7db814eadaa62184c9ebdce22, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733324470803 2024-12-04T15:01:12,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:01:12,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:12,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:01:12,971 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:12,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:01:12,971 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/B is initiating minor compaction (all files) 2024-12-04T15:01:12,971 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/B in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:12,971 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/835107bab2f84af8ac5bc368936de601, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/79b12ab3ab33468abea445e81007b833, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/d4bc9fa710074e61bca64acf7ff29050] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=35.9 K 2024-12-04T15:01:12,972 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 835107bab2f84af8ac5bc368936de601, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733324467828 2024-12-04T15:01:12,973 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 79b12ab3ab33468abea445e81007b833, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733324470154 2024-12-04T15:01:12,974 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting d4bc9fa710074e61bca64acf7ff29050, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733324470814 2024-12-04T15:01:12,995 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:13,004 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#B#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:13,004 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/38c6969167024e859850f017e20a995e is 50, key is test_row_0/B:col10/1733324471951/Put/seqid=0 2024-12-04T15:01:13,015 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204164c0d392984493aa9a5274e675ea677_9bc7edf6120a83a9451ae3a6357eaeb9 store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:13,017 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204164c0d392984493aa9a5274e675ea677_9bc7edf6120a83a9451ae3a6357eaeb9, store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:13,017 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204164c0d392984493aa9a5274e675ea677_9bc7edf6120a83a9451ae3a6357eaeb9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:13,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742421_1597 (size=4469) 2024-12-04T15:01:13,050 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#A#compaction#501 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:13,051 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/b41afaddb4c24c1eac0f9f51783d8a38 is 175, key is test_row_0/A:col10/1733324471951/Put/seqid=0 2024-12-04T15:01:13,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-04T15:01:13,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:13,054 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 9bc7edf6120a83a9451ae3a6357eaeb9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T15:01:13,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=A 2024-12-04T15:01:13,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:13,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=B 2024-12-04T15:01:13,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:13,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=C 2024-12-04T15:01:13,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:13,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742420_1596 (size=12561) 2024-12-04T15:01:13,069 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/38c6969167024e859850f017e20a995e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/38c6969167024e859850f017e20a995e 2024-12-04T15:01:13,072 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/B of 9bc7edf6120a83a9451ae3a6357eaeb9 into 38c6969167024e859850f017e20a995e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:13,073 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:13,073 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/B, priority=13, startTime=1733324472970; duration=0sec 2024-12-04T15:01:13,073 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:01:13,073 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:B 2024-12-04T15:01:13,073 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:13,074 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:13,074 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/C is initiating minor compaction (all files) 2024-12-04T15:01:13,074 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/C in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:13,074 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/c6a2d669e1ed4923b413171488bfc45e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/19c1f1a89a9c47fa8f4f41f038d54cea, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/5b1c17c65b9549c683fdb468ff72c7ce] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=35.9 K 2024-12-04T15:01:13,075 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting c6a2d669e1ed4923b413171488bfc45e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733324467828 2024-12-04T15:01:13,075 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 19c1f1a89a9c47fa8f4f41f038d54cea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733324470154 2024-12-04T15:01:13,075 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b1c17c65b9549c683fdb468ff72c7ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733324470814 2024-12-04T15:01:13,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742422_1598 (size=31515) 2024-12-04T15:01:13,084 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/b41afaddb4c24c1eac0f9f51783d8a38 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/b41afaddb4c24c1eac0f9f51783d8a38 2024-12-04T15:01:13,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f4a39ff6436f4df4a4d42301d776b4b3_9bc7edf6120a83a9451ae3a6357eaeb9 is 50, key is test_row_0/A:col10/1733324471988/Put/seqid=0 2024-12-04T15:01:13,089 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/A of 9bc7edf6120a83a9451ae3a6357eaeb9 into b41afaddb4c24c1eac0f9f51783d8a38(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:13,090 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:13,090 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/A, priority=13, startTime=1733324472967; duration=0sec 2024-12-04T15:01:13,090 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:13,090 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:A 2024-12-04T15:01:13,091 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#C#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:13,091 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/e2c16828bc8b4c6b9d8030da37fe4848 is 50, key is test_row_0/C:col10/1733324471951/Put/seqid=0 2024-12-04T15:01:13,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742423_1599 (size=12304) 2024-12-04T15:01:13,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:13,119 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f4a39ff6436f4df4a4d42301d776b4b3_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f4a39ff6436f4df4a4d42301d776b4b3_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:13,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742424_1600 (size=12561) 2024-12-04T15:01:13,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/331e0c19fba44a93871b09fb1e2a813f, store: [table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:13,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/331e0c19fba44a93871b09fb1e2a813f is 175, key is test_row_0/A:col10/1733324471988/Put/seqid=0 2024-12-04T15:01:13,128 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/e2c16828bc8b4c6b9d8030da37fe4848 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/e2c16828bc8b4c6b9d8030da37fe4848 2024-12-04T15:01:13,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:13,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:13,136 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/C of 9bc7edf6120a83a9451ae3a6357eaeb9 into e2c16828bc8b4c6b9d8030da37fe4848(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:13,136 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:13,136 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/C, priority=13, startTime=1733324472971; duration=0sec 2024-12-04T15:01:13,136 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:13,136 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:C 2024-12-04T15:01:13,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742425_1601 (size=31105) 2024-12-04T15:01:13,157 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/331e0c19fba44a93871b09fb1e2a813f 2024-12-04T15:01:13,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/93807d414475485e896e58f17ebacde4 is 50, key is test_row_0/B:col10/1733324471988/Put/seqid=0 2024-12-04T15:01:13,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742426_1602 (size=12151) 2024-12-04T15:01:13,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:13,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324533192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:13,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324533192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:13,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324533192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:13,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T15:01:13,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:13,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324533300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:13,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:13,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324533300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:13,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:13,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:13,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:13,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324533300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:13,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:13,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324533510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:13,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324533510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:13,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324533510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:13,583 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/93807d414475485e896e58f17ebacde4 2024-12-04T15:01:13,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/0154d4363b504819bc7caac30b480c5d is 50, key is test_row_0/C:col10/1733324471988/Put/seqid=0 2024-12-04T15:01:13,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742427_1603 (size=12151) 2024-12-04T15:01:13,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T15:01:13,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:13,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324533820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:13,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:13,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:13,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324533820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:13,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:13,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:13,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324533820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:13,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:13,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:14,001 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/0154d4363b504819bc7caac30b480c5d 2024-12-04T15:01:14,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/331e0c19fba44a93871b09fb1e2a813f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/331e0c19fba44a93871b09fb1e2a813f 2024-12-04T15:01:14,007 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/331e0c19fba44a93871b09fb1e2a813f, entries=150, sequenceid=232, filesize=30.4 K 2024-12-04T15:01:14,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/93807d414475485e896e58f17ebacde4 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/93807d414475485e896e58f17ebacde4 2024-12-04T15:01:14,011 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/93807d414475485e896e58f17ebacde4, entries=150, sequenceid=232, filesize=11.9 K 2024-12-04T15:01:14,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/0154d4363b504819bc7caac30b480c5d as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/0154d4363b504819bc7caac30b480c5d 2024-12-04T15:01:14,017 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/0154d4363b504819bc7caac30b480c5d, entries=150, sequenceid=232, filesize=11.9 K 2024-12-04T15:01:14,017 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=73.80 KB/75570 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 964ms, sequenceid=232, compaction requested=false 2024-12-04T15:01:14,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:14,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:14,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-04T15:01:14,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-04T15:01:14,020 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-04T15:01:14,020 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4260 sec 2024-12-04T15:01:14,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.4290 sec 2024-12-04T15:01:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:14,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9bc7edf6120a83a9451ae3a6357eaeb9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-04T15:01:14,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=A 2024-12-04T15:01:14,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:14,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=B 2024-12-04T15:01:14,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:14,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=C 2024-12-04T15:01:14,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:14,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120407d15ad13f2847019e021d783c4c02ab_9bc7edf6120a83a9451ae3a6357eaeb9 is 50, key is test_row_0/A:col10/1733324474331/Put/seqid=0 2024-12-04T15:01:14,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742428_1604 (size=14794) 2024-12-04T15:01:14,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324534348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:14,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324534350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:14,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324534351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:14,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324534460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:14,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324534460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324534460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:14,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:14,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43180 deadline: 1733324534550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:14,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:14,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733324534550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:14,551 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 9bc7edf6120a83a9451ae3a6357eaeb9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:01:14,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:14,552 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 9bc7edf6120a83a9451ae3a6357eaeb9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:01:14,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:14,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324534670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:14,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:14,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324534670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:14,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:14,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:14,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324534670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:14,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T15:01:14,740 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:01:14,741 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:01:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-04T15:01:14,742 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T15:01:14,742 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:01:14,743 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:01:14,743 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:01:14,748 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120407d15ad13f2847019e021d783c4c02ab_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120407d15ad13f2847019e021d783c4c02ab_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:14,749 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/437a648d11514b11a105c769ccb1dea0, store: [table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:14,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/437a648d11514b11a105c769ccb1dea0 is 175, key is test_row_0/A:col10/1733324474331/Put/seqid=0 2024-12-04T15:01:14,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742429_1605 (size=39749) 2024-12-04T15:01:14,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T15:01:14,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T15:01:14,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:14,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:14,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:14,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:14,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:14,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324534980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:14,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324534980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324534980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:15,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T15:01:15,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:15,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:15,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:15,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:15,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:15,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T15:01:15,153 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/437a648d11514b11a105c769ccb1dea0 2024-12-04T15:01:15,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/d41d33afb56e444ab2c2cc8993dc6391 is 50, key is test_row_0/B:col10/1733324474331/Put/seqid=0 2024-12-04T15:01:15,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742430_1606 (size=12151) 2024-12-04T15:01:15,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/d41d33afb56e444ab2c2cc8993dc6391 2024-12-04T15:01:15,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/91382da585324870ae2e54dccb0e4f8d is 50, key is test_row_0/C:col10/1733324474331/Put/seqid=0 2024-12-04T15:01:15,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742431_1607 (size=12151) 2024-12-04T15:01:15,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T15:01:15,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:15,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:15,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:15,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:15,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:15,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T15:01:15,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:15,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:15,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:15,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:15,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T15:01:15,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:15,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324535490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:15,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:15,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:15,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324535490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:15,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:15,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:15,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324535490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:15,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:15,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:15,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:15,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:15,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:15,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:15,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:15,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T15:01:15,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:15,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:15,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:15,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:15,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:15,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/91382da585324870ae2e54dccb0e4f8d 2024-12-04T15:01:15,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/437a648d11514b11a105c769ccb1dea0 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/437a648d11514b11a105c769ccb1dea0 2024-12-04T15:01:15,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/437a648d11514b11a105c769ccb1dea0, entries=200, sequenceid=250, filesize=38.8 K 2024-12-04T15:01:15,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/d41d33afb56e444ab2c2cc8993dc6391 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/d41d33afb56e444ab2c2cc8993dc6391 2024-12-04T15:01:15,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/d41d33afb56e444ab2c2cc8993dc6391, entries=150, sequenceid=250, filesize=11.9 K 2024-12-04T15:01:15,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/91382da585324870ae2e54dccb0e4f8d as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/91382da585324870ae2e54dccb0e4f8d 2024-12-04T15:01:15,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/91382da585324870ae2e54dccb0e4f8d, entries=150, sequenceid=250, filesize=11.9 K 2024-12-04T15:01:15,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 1250ms, sequenceid=250, compaction requested=true 2024-12-04T15:01:15,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:15,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:01:15,583 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:15,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:15,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:01:15,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:15,583 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:15,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:01:15,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:01:15,583 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:15,583 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:15,583 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/A is initiating minor compaction (all files) 2024-12-04T15:01:15,583 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/B is initiating minor compaction (all files) 2024-12-04T15:01:15,583 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/A in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:15,583 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/b41afaddb4c24c1eac0f9f51783d8a38, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/331e0c19fba44a93871b09fb1e2a813f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/437a648d11514b11a105c769ccb1dea0] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=100.0 K 2024-12-04T15:01:15,584 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:15,584 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/b41afaddb4c24c1eac0f9f51783d8a38, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/331e0c19fba44a93871b09fb1e2a813f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/437a648d11514b11a105c769ccb1dea0] 2024-12-04T15:01:15,584 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/B in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:15,584 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/38c6969167024e859850f017e20a995e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/93807d414475485e896e58f17ebacde4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/d41d33afb56e444ab2c2cc8993dc6391] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=36.0 K 2024-12-04T15:01:15,585 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 38c6969167024e859850f017e20a995e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733324470814 2024-12-04T15:01:15,585 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting b41afaddb4c24c1eac0f9f51783d8a38, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733324470814 2024-12-04T15:01:15,585 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 93807d414475485e896e58f17ebacde4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733324471984 2024-12-04T15:01:15,585 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 331e0c19fba44a93871b09fb1e2a813f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733324471984 2024-12-04T15:01:15,585 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting d41d33afb56e444ab2c2cc8993dc6391, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733324473146 2024-12-04T15:01:15,586 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 437a648d11514b11a105c769ccb1dea0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733324473146 2024-12-04T15:01:15,589 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:15,591 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#B#compaction#511 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:15,591 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/ea0e54d4e5b6404a89d8fc8f281d19cf is 50, key is test_row_0/B:col10/1733324474331/Put/seqid=0 2024-12-04T15:01:15,593 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204b205c8da40454a10854979605cb95e34_9bc7edf6120a83a9451ae3a6357eaeb9 store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:15,594 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204b205c8da40454a10854979605cb95e34_9bc7edf6120a83a9451ae3a6357eaeb9, store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:15,594 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b205c8da40454a10854979605cb95e34_9bc7edf6120a83a9451ae3a6357eaeb9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:15,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742433_1609 (size=4469) 2024-12-04T15:01:15,612 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#A#compaction#510 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:15,613 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/b3216f2b5e884bd0ba7ae1aca4eac9fc is 175, key is test_row_0/A:col10/1733324474331/Put/seqid=0 2024-12-04T15:01:15,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742432_1608 (size=12663) 2024-12-04T15:01:15,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742434_1610 (size=31617) 2024-12-04T15:01:15,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T15:01:15,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:15,660 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 9bc7edf6120a83a9451ae3a6357eaeb9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-04T15:01:15,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=A 2024-12-04T15:01:15,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:15,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=B 2024-12-04T15:01:15,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:15,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=C 2024-12-04T15:01:15,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:15,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120499bafa0c989143c9ba79b7d892ebf53b_9bc7edf6120a83a9451ae3a6357eaeb9 is 50, key is test_row_0/A:col10/1733324474350/Put/seqid=0 2024-12-04T15:01:15,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742435_1611 (size=12454) 2024-12-04T15:01:15,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T15:01:16,021 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/ea0e54d4e5b6404a89d8fc8f281d19cf as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/ea0e54d4e5b6404a89d8fc8f281d19cf 2024-12-04T15:01:16,025 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/B of 9bc7edf6120a83a9451ae3a6357eaeb9 into ea0e54d4e5b6404a89d8fc8f281d19cf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:16,025 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:16,025 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/B, priority=13, startTime=1733324475583; duration=0sec 2024-12-04T15:01:16,026 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:01:16,026 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:B 2024-12-04T15:01:16,026 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:16,027 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/b3216f2b5e884bd0ba7ae1aca4eac9fc as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/b3216f2b5e884bd0ba7ae1aca4eac9fc 2024-12-04T15:01:16,027 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:16,027 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/C is initiating minor compaction (all files) 2024-12-04T15:01:16,027 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/C in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:16,027 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/e2c16828bc8b4c6b9d8030da37fe4848, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/0154d4363b504819bc7caac30b480c5d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/91382da585324870ae2e54dccb0e4f8d] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=36.0 K 2024-12-04T15:01:16,037 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting e2c16828bc8b4c6b9d8030da37fe4848, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733324470814 2024-12-04T15:01:16,037 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 0154d4363b504819bc7caac30b480c5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733324471984 2024-12-04T15:01:16,038 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 91382da585324870ae2e54dccb0e4f8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733324473146 2024-12-04T15:01:16,042 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/A of 9bc7edf6120a83a9451ae3a6357eaeb9 into b3216f2b5e884bd0ba7ae1aca4eac9fc(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:16,042 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:16,042 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/A, priority=13, startTime=1733324475583; duration=0sec 2024-12-04T15:01:16,042 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:16,042 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:A 2024-12-04T15:01:16,045 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#C#compaction#513 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:16,045 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/6ed2ed9c44ea4f96bf9139d7506c7588 is 50, key is test_row_0/C:col10/1733324474331/Put/seqid=0 2024-12-04T15:01:16,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:16,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742436_1612 (size=12663) 2024-12-04T15:01:16,073 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120499bafa0c989143c9ba79b7d892ebf53b_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120499bafa0c989143c9ba79b7d892ebf53b_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:16,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/f7987f58f98c4ee7ba99f08291861ec2, store: [table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:16,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/f7987f58f98c4ee7ba99f08291861ec2 is 175, key is test_row_0/A:col10/1733324474350/Put/seqid=0 2024-12-04T15:01:16,082 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/6ed2ed9c44ea4f96bf9139d7506c7588 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/6ed2ed9c44ea4f96bf9139d7506c7588 2024-12-04T15:01:16,087 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/C of 9bc7edf6120a83a9451ae3a6357eaeb9 into 6ed2ed9c44ea4f96bf9139d7506c7588(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:16,087 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:16,087 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/C, priority=13, startTime=1733324475583; duration=0sec 2024-12-04T15:01:16,087 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:16,087 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:C 2024-12-04T15:01:16,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742437_1613 (size=31255) 2024-12-04T15:01:16,092 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/f7987f58f98c4ee7ba99f08291861ec2 2024-12-04T15:01:16,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/b4b958945bcc47308cfdd89fb694c015 is 50, key is test_row_0/B:col10/1733324474350/Put/seqid=0 2024-12-04T15:01:16,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742438_1614 (size=12301) 2024-12-04T15:01:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:16,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:16,503 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/b4b958945bcc47308cfdd89fb694c015 2024-12-04T15:01:16,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/e5e7169813904021bd31ca972739366b is 50, key is test_row_0/C:col10/1733324474350/Put/seqid=0 2024-12-04T15:01:16,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742439_1615 (size=12301) 2024-12-04T15:01:16,511 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/e5e7169813904021bd31ca972739366b 2024-12-04T15:01:16,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/f7987f58f98c4ee7ba99f08291861ec2 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/f7987f58f98c4ee7ba99f08291861ec2 2024-12-04T15:01:16,517 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/f7987f58f98c4ee7ba99f08291861ec2, entries=150, sequenceid=267, filesize=30.5 K 2024-12-04T15:01:16,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/b4b958945bcc47308cfdd89fb694c015 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/b4b958945bcc47308cfdd89fb694c015 2024-12-04T15:01:16,520 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/b4b958945bcc47308cfdd89fb694c015, entries=150, sequenceid=267, filesize=12.0 K 2024-12-04T15:01:16,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/e5e7169813904021bd31ca972739366b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/e5e7169813904021bd31ca972739366b 2024-12-04T15:01:16,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324536519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:16,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324536520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:16,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324536521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,523 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/e5e7169813904021bd31ca972739366b, entries=150, sequenceid=267, filesize=12.0 K 2024-12-04T15:01:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:16,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:16,524 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 864ms, sequenceid=267, compaction requested=false 2024-12-04T15:01:16,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:16,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:16,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-04T15:01:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-04T15:01:16,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-04T15:01:16,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7820 sec 2024-12-04T15:01:16,528 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.7860 sec 2024-12-04T15:01:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:16,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9bc7edf6120a83a9451ae3a6357eaeb9 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-04T15:01:16,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=A 2024-12-04T15:01:16,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:16,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=B 2024-12-04T15:01:16,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:16,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=C 2024-12-04T15:01:16,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:16,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d79bdc0119274369b619f7f00c3a9dfa_9bc7edf6120a83a9451ae3a6357eaeb9 is 50, key is test_row_0/A:col10/1733324476513/Put/seqid=0 2024-12-04T15:01:16,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742440_1616 (size=12454) 2024-12-04T15:01:16,644 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:16,647 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d79bdc0119274369b619f7f00c3a9dfa_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d79bdc0119274369b619f7f00c3a9dfa_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:16,648 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/e2546e7b4f6b4d9e87509633152b85e3, store: [table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:16,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/e2546e7b4f6b4d9e87509633152b85e3 is 175, key is test_row_0/A:col10/1733324476513/Put/seqid=0 2024-12-04T15:01:16,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324536647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:16,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:16,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742441_1617 (size=31255) 2024-12-04T15:01:16,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324536650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:16,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:16,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324536650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:16,659 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/e2546e7b4f6b4d9e87509633152b85e3 2024-12-04T15:01:16,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/1bd9ce90782d4476944df36790b5f082 is 50, key is test_row_0/B:col10/1733324476513/Put/seqid=0 2024-12-04T15:01:16,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742442_1618 (size=12301) 2024-12-04T15:01:16,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:16,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324536760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:16,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324536760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:16,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324536760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T15:01:16,900 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:01:16,900 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:01:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-04T15:01:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T15:01:16,901 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:01:16,902 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:01:16,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:01:16,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324536970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324536970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324536970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:16,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:16,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T15:01:17,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T15:01:17,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:17,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:17,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:17,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:17,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:17,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/1bd9ce90782d4476944df36790b5f082 2024-12-04T15:01:17,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/73c1c5ec9ee540879a61c2ec22b08964 is 50, key is test_row_0/C:col10/1733324476513/Put/seqid=0 2024-12-04T15:01:17,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742443_1619 (size=12301) 2024-12-04T15:01:17,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T15:01:17,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:17,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:17,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:17,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:17,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T15:01:17,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:17,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324537280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324537280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:17,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324537280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:17,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:17,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:17,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:17,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T15:01:17,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:17,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:17,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:17,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:17,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:17,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/73c1c5ec9ee540879a61c2ec22b08964 2024-12-04T15:01:17,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/e2546e7b4f6b4d9e87509633152b85e3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/e2546e7b4f6b4d9e87509633152b85e3 2024-12-04T15:01:17,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/e2546e7b4f6b4d9e87509633152b85e3, entries=150, sequenceid=286, filesize=30.5 K 2024-12-04T15:01:17,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/1bd9ce90782d4476944df36790b5f082 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/1bd9ce90782d4476944df36790b5f082 2024-12-04T15:01:17,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/1bd9ce90782d4476944df36790b5f082, entries=150, sequenceid=286, filesize=12.0 K 2024-12-04T15:01:17,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/73c1c5ec9ee540879a61c2ec22b08964 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/73c1c5ec9ee540879a61c2ec22b08964 2024-12-04T15:01:17,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/73c1c5ec9ee540879a61c2ec22b08964, entries=150, sequenceid=286, filesize=12.0 K 2024-12-04T15:01:17,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 861ms, sequenceid=286, compaction requested=true 2024-12-04T15:01:17,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:17,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:01:17,492 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:17,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:17,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:01:17,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:17,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:01:17,492 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:17,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:01:17,494 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:17,494 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:17,494 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/A is initiating minor compaction (all files) 2024-12-04T15:01:17,494 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/B is initiating minor compaction (all files) 2024-12-04T15:01:17,494 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/B in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:17,494 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/A in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:17,495 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/ea0e54d4e5b6404a89d8fc8f281d19cf, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/b4b958945bcc47308cfdd89fb694c015, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/1bd9ce90782d4476944df36790b5f082] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=36.4 K 2024-12-04T15:01:17,495 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/b3216f2b5e884bd0ba7ae1aca4eac9fc, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/f7987f58f98c4ee7ba99f08291861ec2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/e2546e7b4f6b4d9e87509633152b85e3] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=91.9 K 2024-12-04T15:01:17,495 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:17,495 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/b3216f2b5e884bd0ba7ae1aca4eac9fc, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/f7987f58f98c4ee7ba99f08291861ec2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/e2546e7b4f6b4d9e87509633152b85e3] 2024-12-04T15:01:17,495 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting ea0e54d4e5b6404a89d8fc8f281d19cf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733324473146 2024-12-04T15:01:17,495 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3216f2b5e884bd0ba7ae1aca4eac9fc, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733324473146 2024-12-04T15:01:17,495 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting b4b958945bcc47308cfdd89fb694c015, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733324474336 2024-12-04T15:01:17,495 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7987f58f98c4ee7ba99f08291861ec2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733324474336 2024-12-04T15:01:17,495 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bd9ce90782d4476944df36790b5f082, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733324476513 2024-12-04T15:01:17,495 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2546e7b4f6b4d9e87509633152b85e3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733324476513 2024-12-04T15:01:17,500 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:17,501 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#B#compaction#519 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:17,502 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204f1ca0f5a211b4a6fb2139fcaca2007c2_9bc7edf6120a83a9451ae3a6357eaeb9 store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:17,502 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/7d7ba7aea7144034a705af1b6c84c08c is 50, key is test_row_0/B:col10/1733324476513/Put/seqid=0 2024-12-04T15:01:17,503 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204f1ca0f5a211b4a6fb2139fcaca2007c2_9bc7edf6120a83a9451ae3a6357eaeb9, store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:17,503 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f1ca0f5a211b4a6fb2139fcaca2007c2_9bc7edf6120a83a9451ae3a6357eaeb9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:17,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742444_1620 (size=12915) 2024-12-04T15:01:17,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742445_1621 (size=4469) 2024-12-04T15:01:17,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T15:01:17,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:17,510 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 9bc7edf6120a83a9451ae3a6357eaeb9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-04T15:01:17,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=A 2024-12-04T15:01:17,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:17,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=B 2024-12-04T15:01:17,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:17,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=C 2024-12-04T15:01:17,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:17,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204afe73506933e43ad8b9a338b96436c87_9bc7edf6120a83a9451ae3a6357eaeb9 is 50, key is test_row_0/A:col10/1733324476634/Put/seqid=0 2024-12-04T15:01:17,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742446_1622 (size=12454) 2024-12-04T15:01:17,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T15:01:17,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:17,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:17,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:17,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324537829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:17,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:17,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:17,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:17,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:17,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:17,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324537829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:17,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324537829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:17,908 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#A#compaction#520 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:17,908 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/d6197ed4d5a14e2c8bb1a0dd245db1c2 is 175, key is test_row_0/A:col10/1733324476513/Put/seqid=0 2024-12-04T15:01:17,910 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/7d7ba7aea7144034a705af1b6c84c08c as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/7d7ba7aea7144034a705af1b6c84c08c 2024-12-04T15:01:17,914 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/B of 9bc7edf6120a83a9451ae3a6357eaeb9 into 7d7ba7aea7144034a705af1b6c84c08c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:17,914 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:17,914 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/B, priority=13, startTime=1733324477492; duration=0sec 2024-12-04T15:01:17,914 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:01:17,914 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:B 2024-12-04T15:01:17,914 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:17,915 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:17,915 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/C is initiating minor compaction (all files) 2024-12-04T15:01:17,915 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/C in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:17,915 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/6ed2ed9c44ea4f96bf9139d7506c7588, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/e5e7169813904021bd31ca972739366b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/73c1c5ec9ee540879a61c2ec22b08964] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=36.4 K 2024-12-04T15:01:17,915 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ed2ed9c44ea4f96bf9139d7506c7588, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733324473146 2024-12-04T15:01:17,916 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting e5e7169813904021bd31ca972739366b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733324474336 2024-12-04T15:01:17,916 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 73c1c5ec9ee540879a61c2ec22b08964, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733324476513 2024-12-04T15:01:17,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:17,923 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204afe73506933e43ad8b9a338b96436c87_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204afe73506933e43ad8b9a338b96436c87_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:17,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/a37e684930d9462db691a6d662e39347, store: [table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:17,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/a37e684930d9462db691a6d662e39347 is 175, key is test_row_0/A:col10/1733324476634/Put/seqid=0 2024-12-04T15:01:17,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742447_1623 (size=31869) 2024-12-04T15:01:17,932 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#C#compaction#522 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:17,933 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/bd05cc08962844f0b711fc2dffc34da4 is 50, key is test_row_0/C:col10/1733324476513/Put/seqid=0 2024-12-04T15:01:17,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742448_1624 (size=31255) 2024-12-04T15:01:17,934 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/a37e684930d9462db691a6d662e39347 2024-12-04T15:01:17,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324537940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:17,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324537940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:17,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324537940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:17,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:17,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:17,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:17,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/def460423ccc4b54ba7c7482e47be509 is 50, key is test_row_0/B:col10/1733324476634/Put/seqid=0 2024-12-04T15:01:17,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742449_1625 (size=12915) 2024-12-04T15:01:17,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742450_1626 (size=12301) 2024-12-04T15:01:17,953 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/def460423ccc4b54ba7c7482e47be509 2024-12-04T15:01:17,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/fb39dd9cc1c7463b85028db289638453 is 50, key is test_row_0/C:col10/1733324476634/Put/seqid=0 2024-12-04T15:01:17,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742451_1627 (size=12301) 2024-12-04T15:01:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T15:01:18,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324538150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:18,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324538150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:18,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:18,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:18,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324538150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:18,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:18,335 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/d6197ed4d5a14e2c8bb1a0dd245db1c2 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d6197ed4d5a14e2c8bb1a0dd245db1c2 2024-12-04T15:01:18,339 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/A of 9bc7edf6120a83a9451ae3a6357eaeb9 into d6197ed4d5a14e2c8bb1a0dd245db1c2(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:18,339 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:18,339 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/A, priority=13, startTime=1733324477492; duration=0sec 2024-12-04T15:01:18,339 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:18,339 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:A 2024-12-04T15:01:18,353 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/bd05cc08962844f0b711fc2dffc34da4 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/bd05cc08962844f0b711fc2dffc34da4 2024-12-04T15:01:18,357 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/C of 9bc7edf6120a83a9451ae3a6357eaeb9 into bd05cc08962844f0b711fc2dffc34da4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:18,357 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:18,357 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/C, priority=13, startTime=1733324477492; duration=0sec 2024-12-04T15:01:18,357 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:18,357 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:C 2024-12-04T15:01:18,362 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/fb39dd9cc1c7463b85028db289638453 2024-12-04T15:01:18,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/a37e684930d9462db691a6d662e39347 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/a37e684930d9462db691a6d662e39347 2024-12-04T15:01:18,368 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/a37e684930d9462db691a6d662e39347, entries=150, sequenceid=303, filesize=30.5 K 2024-12-04T15:01:18,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/def460423ccc4b54ba7c7482e47be509 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/def460423ccc4b54ba7c7482e47be509 2024-12-04T15:01:18,372 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/def460423ccc4b54ba7c7482e47be509, entries=150, sequenceid=303, filesize=12.0 K 2024-12-04T15:01:18,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/fb39dd9cc1c7463b85028db289638453 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/fb39dd9cc1c7463b85028db289638453 2024-12-04T15:01:18,375 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/fb39dd9cc1c7463b85028db289638453, entries=150, sequenceid=303, filesize=12.0 K 2024-12-04T15:01:18,376 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=93.93 KB/96180 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 866ms, sequenceid=303, compaction requested=false 2024-12-04T15:01:18,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:18,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:18,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-04T15:01:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-04T15:01:18,378 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-04T15:01:18,378 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4750 sec 2024-12-04T15:01:18,380 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.4790 sec 2024-12-04T15:01:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:18,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9bc7edf6120a83a9451ae3a6357eaeb9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-04T15:01:18,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=A 2024-12-04T15:01:18,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:18,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=B 2024-12-04T15:01:18,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:18,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=C 2024-12-04T15:01:18,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:18,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204ade3813fe9f542e7b737b5c214d36be3_9bc7edf6120a83a9451ae3a6357eaeb9 is 50, key is test_row_0/A:col10/1733324478462/Put/seqid=0 2024-12-04T15:01:18,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742452_1628 (size=14994) 2024-12-04T15:01:18,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324538500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:18,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:18,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324538501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:18,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:18,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:18,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324538501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:18,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:18,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324538610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:18,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324538610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324538610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:18,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:18,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324538820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:18,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324538820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324538820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:18,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:18,872 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:18,875 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204ade3813fe9f542e7b737b5c214d36be3_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ade3813fe9f542e7b737b5c214d36be3_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:18,875 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/bad394ceb43045ec8624a528091ab1d1, store: [table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:18,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/bad394ceb43045ec8624a528091ab1d1 is 175, key is test_row_0/A:col10/1733324478462/Put/seqid=0 2024-12-04T15:01:18,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742453_1629 (size=39949) 2024-12-04T15:01:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T15:01:19,050 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:01:19,050 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:01:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-04T15:01:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T15:01:19,052 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:01:19,052 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:01:19,052 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:01:19,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324539130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:19,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324539131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:19,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:19,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324539131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T15:01:19,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T15:01:19,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:19,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:19,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:19,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:19,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:19,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:19,279 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/bad394ceb43045ec8624a528091ab1d1 2024-12-04T15:01:19,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/0099da0038a743b4bb6b7feeab932f91 is 50, key is test_row_0/B:col10/1733324478462/Put/seqid=0 2024-12-04T15:01:19,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742454_1630 (size=12301) 2024-12-04T15:01:19,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/0099da0038a743b4bb6b7feeab932f91 2024-12-04T15:01:19,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/f508b9b02aba41ecaac1e32dac234670 is 50, key is test_row_0/C:col10/1733324478462/Put/seqid=0 2024-12-04T15:01:19,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742455_1631 (size=12301) 2024-12-04T15:01:19,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/f508b9b02aba41ecaac1e32dac234670 2024-12-04T15:01:19,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/bad394ceb43045ec8624a528091ab1d1 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/bad394ceb43045ec8624a528091ab1d1 2024-12-04T15:01:19,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/bad394ceb43045ec8624a528091ab1d1, entries=200, sequenceid=326, filesize=39.0 K 2024-12-04T15:01:19,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T15:01:19,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:19,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:19,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:19,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:19,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:19,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/0099da0038a743b4bb6b7feeab932f91 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/0099da0038a743b4bb6b7feeab932f91 2024-12-04T15:01:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:19,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/0099da0038a743b4bb6b7feeab932f91, entries=150, sequenceid=326, filesize=12.0 K 2024-12-04T15:01:19,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/f508b9b02aba41ecaac1e32dac234670 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/f508b9b02aba41ecaac1e32dac234670 2024-12-04T15:01:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T15:01:19,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/f508b9b02aba41ecaac1e32dac234670, entries=150, sequenceid=326, filesize=12.0 K 2024-12-04T15:01:19,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 911ms, sequenceid=326, compaction requested=true 2024-12-04T15:01:19,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:19,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:01:19,373 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:19,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:19,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:01:19,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:19,373 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:19,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:01:19,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:01:19,374 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:19,374 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:19,374 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/B is initiating minor compaction (all files) 2024-12-04T15:01:19,374 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/A is initiating minor compaction (all files) 2024-12-04T15:01:19,374 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/B in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:19,374 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/A in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:19,374 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/7d7ba7aea7144034a705af1b6c84c08c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/def460423ccc4b54ba7c7482e47be509, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/0099da0038a743b4bb6b7feeab932f91] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=36.6 K 2024-12-04T15:01:19,374 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d6197ed4d5a14e2c8bb1a0dd245db1c2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/a37e684930d9462db691a6d662e39347, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/bad394ceb43045ec8624a528091ab1d1] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=100.7 K 2024-12-04T15:01:19,374 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:19,374 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d6197ed4d5a14e2c8bb1a0dd245db1c2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/a37e684930d9462db691a6d662e39347, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/bad394ceb43045ec8624a528091ab1d1] 2024-12-04T15:01:19,374 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d7ba7aea7144034a705af1b6c84c08c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733324476513 2024-12-04T15:01:19,375 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6197ed4d5a14e2c8bb1a0dd245db1c2, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733324476513 2024-12-04T15:01:19,375 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting def460423ccc4b54ba7c7482e47be509, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733324476634 2024-12-04T15:01:19,376 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting a37e684930d9462db691a6d662e39347, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733324476634 2024-12-04T15:01:19,376 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 0099da0038a743b4bb6b7feeab932f91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733324477796 2024-12-04T15:01:19,377 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting bad394ceb43045ec8624a528091ab1d1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733324477796 2024-12-04T15:01:19,387 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:19,390 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#B#compaction#529 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:19,390 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/89ab73dae04a4658b2c65b645d7b28d3 is 50, key is test_row_0/B:col10/1733324478462/Put/seqid=0 2024-12-04T15:01:19,390 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204c25ae810cf974c86b5cd34babf27d818_9bc7edf6120a83a9451ae3a6357eaeb9 store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:19,392 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204c25ae810cf974c86b5cd34babf27d818_9bc7edf6120a83a9451ae3a6357eaeb9, store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:19,392 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c25ae810cf974c86b5cd34babf27d818_9bc7edf6120a83a9451ae3a6357eaeb9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:19,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742456_1632 (size=13017) 2024-12-04T15:01:19,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742457_1633 (size=4469) 2024-12-04T15:01:19,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T15:01:19,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:19,509 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 9bc7edf6120a83a9451ae3a6357eaeb9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T15:01:19,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=A 2024-12-04T15:01:19,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:19,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=B 2024-12-04T15:01:19,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:19,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=C 2024-12-04T15:01:19,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:19,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204dfd0135866d2460d8b59d311d1a13984_9bc7edf6120a83a9451ae3a6357eaeb9 is 50, key is test_row_0/A:col10/1733324478469/Put/seqid=0 2024-12-04T15:01:19,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742458_1634 (size=12454) 2024-12-04T15:01:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:19,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:19,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:19,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324539664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:19,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:19,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:19,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324539666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:19,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:19,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:19,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324539667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:19,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:19,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T15:01:19,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:19,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324539770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:19,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:19,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:19,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324539780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:19,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:19,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324539780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:19,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:19,824 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/89ab73dae04a4658b2c65b645d7b28d3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/89ab73dae04a4658b2c65b645d7b28d3 2024-12-04T15:01:19,827 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/B of 9bc7edf6120a83a9451ae3a6357eaeb9 into 89ab73dae04a4658b2c65b645d7b28d3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:19,827 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:19,827 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/B, priority=13, startTime=1733324479373; duration=0sec 2024-12-04T15:01:19,827 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:01:19,827 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:B 2024-12-04T15:01:19,828 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:19,828 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:19,828 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/C is initiating minor compaction (all files) 2024-12-04T15:01:19,828 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/C in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:19,828 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/bd05cc08962844f0b711fc2dffc34da4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/fb39dd9cc1c7463b85028db289638453, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/f508b9b02aba41ecaac1e32dac234670] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=36.6 K 2024-12-04T15:01:19,828 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting bd05cc08962844f0b711fc2dffc34da4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733324476513 2024-12-04T15:01:19,829 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting fb39dd9cc1c7463b85028db289638453, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733324476634 2024-12-04T15:01:19,829 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting f508b9b02aba41ecaac1e32dac234670, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733324477796 2024-12-04T15:01:19,832 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#C#compaction#531 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:19,833 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/dc04cf16fc754987a54ec44748ecd37e is 50, key is test_row_0/C:col10/1733324478462/Put/seqid=0 2024-12-04T15:01:19,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742459_1635 (size=13017) 2024-12-04T15:01:19,848 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#A#compaction#528 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:19,849 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/0c553aef50ec43448c6f825e9f41c96b is 175, key is test_row_0/A:col10/1733324478462/Put/seqid=0 2024-12-04T15:01:19,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742460_1636 (size=31971) 2024-12-04T15:01:19,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:19,921 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204dfd0135866d2460d8b59d311d1a13984_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204dfd0135866d2460d8b59d311d1a13984_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:19,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/ed1392d627434a39839fca0948b2f1b6, store: [table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:19,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/ed1392d627434a39839fca0948b2f1b6 is 175, key is test_row_0/A:col10/1733324478469/Put/seqid=0 2024-12-04T15:01:19,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742461_1637 (size=31255) 2024-12-04T15:01:19,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324539980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:19,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:19,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:19,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324539990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:19,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324539990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:20,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T15:01:20,239 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/dc04cf16fc754987a54ec44748ecd37e as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/dc04cf16fc754987a54ec44748ecd37e 2024-12-04T15:01:20,243 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/C of 9bc7edf6120a83a9451ae3a6357eaeb9 into dc04cf16fc754987a54ec44748ecd37e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:20,243 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:20,243 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/C, priority=13, startTime=1733324479373; duration=0sec 2024-12-04T15:01:20,243 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:20,243 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:C 2024-12-04T15:01:20,255 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/0c553aef50ec43448c6f825e9f41c96b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/0c553aef50ec43448c6f825e9f41c96b 2024-12-04T15:01:20,259 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/A of 9bc7edf6120a83a9451ae3a6357eaeb9 into 0c553aef50ec43448c6f825e9f41c96b(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:20,259 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:20,259 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/A, priority=13, startTime=1733324479373; duration=0sec 2024-12-04T15:01:20,259 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:20,259 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:A 2024-12-04T15:01:20,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:20,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324540290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:20,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:20,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:20,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:20,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324540300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:20,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324540300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:20,325 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/ed1392d627434a39839fca0948b2f1b6 2024-12-04T15:01:20,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/f88c592b53be4816a42a53e8a2fde040 is 50, key is test_row_0/B:col10/1733324478469/Put/seqid=0 2024-12-04T15:01:20,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742462_1638 (size=12301) 2024-12-04T15:01:20,733 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/f88c592b53be4816a42a53e8a2fde040 2024-12-04T15:01:20,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/7d4b7811e7024694a33f9e34e9058cb4 is 50, key is test_row_0/C:col10/1733324478469/Put/seqid=0 2024-12-04T15:01:20,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742463_1639 (size=12301) 2024-12-04T15:01:20,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324540800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:20,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:20,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324540810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:20,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:20,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:20,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:20,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324540810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:20,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:20,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:20,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:20,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:21,141 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/7d4b7811e7024694a33f9e34e9058cb4 2024-12-04T15:01:21,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/ed1392d627434a39839fca0948b2f1b6 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/ed1392d627434a39839fca0948b2f1b6 2024-12-04T15:01:21,146 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/ed1392d627434a39839fca0948b2f1b6, entries=150, sequenceid=338, filesize=30.5 K 2024-12-04T15:01:21,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/f88c592b53be4816a42a53e8a2fde040 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/f88c592b53be4816a42a53e8a2fde040 2024-12-04T15:01:21,149 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/f88c592b53be4816a42a53e8a2fde040, entries=150, sequenceid=338, filesize=12.0 K 2024-12-04T15:01:21,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/7d4b7811e7024694a33f9e34e9058cb4 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/7d4b7811e7024694a33f9e34e9058cb4 2024-12-04T15:01:21,152 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/7d4b7811e7024694a33f9e34e9058cb4, entries=150, sequenceid=338, filesize=12.0 K 2024-12-04T15:01:21,153 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 1643ms, sequenceid=338, compaction requested=false 2024-12-04T15:01:21,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:21,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:21,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-04T15:01:21,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-04T15:01:21,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-04T15:01:21,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1020 sec 2024-12-04T15:01:21,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.1050 sec 2024-12-04T15:01:21,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T15:01:21,201 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:01:21,201 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:01:21,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-04T15:01:21,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T15:01:21,202 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:01:21,203 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:01:21,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:01:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T15:01:21,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-04T15:01:21,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:21,355 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 9bc7edf6120a83a9451ae3a6357eaeb9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T15:01:21,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=A 2024-12-04T15:01:21,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:21,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=B 2024-12-04T15:01:21,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:21,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=C 2024-12-04T15:01:21,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:21,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a6c6077dde674ad3b7134e034f7e5b73_9bc7edf6120a83a9451ae3a6357eaeb9 is 50, key is test_row_0/A:col10/1733324479665/Put/seqid=0 2024-12-04T15:01:21,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742464_1640 (size=12454) 2024-12-04T15:01:21,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:21,380 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a6c6077dde674ad3b7134e034f7e5b73_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a6c6077dde674ad3b7134e034f7e5b73_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:21,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/fe1942e1b7b44a53be719156b0aae2e2, store: [table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:21,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/fe1942e1b7b44a53be719156b0aae2e2 is 175, key is test_row_0/A:col10/1733324479665/Put/seqid=0 2024-12-04T15:01:21,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742465_1641 (size=31255) 2024-12-04T15:01:21,391 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=361, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/fe1942e1b7b44a53be719156b0aae2e2 2024-12-04T15:01:21,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/84f4e9d5f2374a14a3235c5aa0e76686 is 50, key is test_row_0/B:col10/1733324479665/Put/seqid=0 2024-12-04T15:01:21,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742466_1642 (size=12301) 2024-12-04T15:01:21,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T15:01:21,805 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/84f4e9d5f2374a14a3235c5aa0e76686 2024-12-04T15:01:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:21,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:21,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/1fb911bbd50c4bd2adbeb328700b991f is 50, key is test_row_0/C:col10/1733324479665/Put/seqid=0 2024-12-04T15:01:21,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742467_1643 (size=12301) 2024-12-04T15:01:21,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T15:01:21,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324541832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:21,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324541833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:21,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324541833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:21,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324541940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:21,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324541940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:21,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324541940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:22,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324542150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:22,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324542150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324542150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:22,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:22,217 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/1fb911bbd50c4bd2adbeb328700b991f 2024-12-04T15:01:22,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/fe1942e1b7b44a53be719156b0aae2e2 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/fe1942e1b7b44a53be719156b0aae2e2 2024-12-04T15:01:22,223 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/fe1942e1b7b44a53be719156b0aae2e2, entries=150, sequenceid=361, filesize=30.5 K 2024-12-04T15:01:22,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/84f4e9d5f2374a14a3235c5aa0e76686 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/84f4e9d5f2374a14a3235c5aa0e76686 2024-12-04T15:01:22,226 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/84f4e9d5f2374a14a3235c5aa0e76686, entries=150, sequenceid=361, filesize=12.0 K 2024-12-04T15:01:22,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/1fb911bbd50c4bd2adbeb328700b991f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/1fb911bbd50c4bd2adbeb328700b991f 2024-12-04T15:01:22,231 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/1fb911bbd50c4bd2adbeb328700b991f, entries=150, sequenceid=361, filesize=12.0 K 2024-12-04T15:01:22,234 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 879ms, sequenceid=361, compaction requested=true 2024-12-04T15:01:22,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:22,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:22,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-04T15:01:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-04T15:01:22,236 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-04T15:01:22,236 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0320 sec 2024-12-04T15:01:22,238 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.0360 sec 2024-12-04T15:01:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T15:01:22,340 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:01:22,340 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T15:01:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-12-04T15:01:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-04T15:01:22,342 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T15:01:22,342 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T15:01:22,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T15:01:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-04T15:01:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:22,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9bc7edf6120a83a9451ae3a6357eaeb9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-04T15:01:22,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=A 2024-12-04T15:01:22,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:22,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=B 2024-12-04T15:01:22,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:22,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=C 2024-12-04T15:01:22,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:22,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120413ce2c73b382465684ae73b5b7f5bdfd_9bc7edf6120a83a9451ae3a6357eaeb9 is 50, key is test_row_0/A:col10/1733324482462/Put/seqid=0 2024-12-04T15:01:22,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742468_1644 (size=14994) 2024-12-04T15:01:22,480 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:22,482 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120413ce2c73b382465684ae73b5b7f5bdfd_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120413ce2c73b382465684ae73b5b7f5bdfd_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:22,483 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/5581b46df56d4d35b3812c1ded23db32, store: [table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:22,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/5581b46df56d4d35b3812c1ded23db32 is 175, key is test_row_0/A:col10/1733324482462/Put/seqid=0 2024-12-04T15:01:22,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324542483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:22,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:22,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742469_1645 (size=39949) 2024-12-04T15:01:22,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324542486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:22,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324542486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:22,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:22,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:22,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-04T15:01:22,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:22,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:22,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:22,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:22,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:22,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324542590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:22,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324542600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:22,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324542600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:22,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:22,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-04T15:01:22,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:22,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:22,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:22,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:22,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-04T15:01:22,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-04T15:01:22,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:22,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:22,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:22,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:22,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:22,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324542800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:22,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:22,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324542810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:22,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:22,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324542810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:22,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:22,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:22,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:22,887 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=377, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/5581b46df56d4d35b3812c1ded23db32 2024-12-04T15:01:22,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/58b6f254ee5f49fa90a31842da574cc3 is 50, key is test_row_0/B:col10/1733324482462/Put/seqid=0 2024-12-04T15:01:22,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742470_1646 (size=12301) 2024-12-04T15:01:22,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-04T15:01:22,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:22,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:22,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:22,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:22,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-04T15:01:23,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-04T15:01:23,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:23,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:23,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:23,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:23,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:23,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324543110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:23,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:23,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324543120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324543120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:23,226 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T15:01:23,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-04T15:01:23,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:23,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:23,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:23,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:23,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:01:23,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/58b6f254ee5f49fa90a31842da574cc3 2024-12-04T15:01:23,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/15cfd777aaac4002a450af2f0db5c355 is 50, key is test_row_0/C:col10/1733324482462/Put/seqid=0 2024-12-04T15:01:23,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742471_1647 (size=12301) 2024-12-04T15:01:23,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/15cfd777aaac4002a450af2f0db5c355 2024-12-04T15:01:23,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/5581b46df56d4d35b3812c1ded23db32 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/5581b46df56d4d35b3812c1ded23db32 2024-12-04T15:01:23,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/5581b46df56d4d35b3812c1ded23db32, entries=200, sequenceid=377, filesize=39.0 K 2024-12-04T15:01:23,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/58b6f254ee5f49fa90a31842da574cc3 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/58b6f254ee5f49fa90a31842da574cc3 2024-12-04T15:01:23,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/58b6f254ee5f49fa90a31842da574cc3, entries=150, sequenceid=377, filesize=12.0 K 2024-12-04T15:01:23,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/15cfd777aaac4002a450af2f0db5c355 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/15cfd777aaac4002a450af2f0db5c355 2024-12-04T15:01:23,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/15cfd777aaac4002a450af2f0db5c355, entries=150, sequenceid=377, filesize=12.0 K 2024-12-04T15:01:23,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 881ms, sequenceid=377, compaction requested=true 2024-12-04T15:01:23,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:23,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:01:23,344 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T15:01:23,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:23,344 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T15:01:23,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:01:23,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:23,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:01:23,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:01:23,345 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T15:01:23,345 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/A is initiating minor compaction (all files) 2024-12-04T15:01:23,345 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/A in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:23,345 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/0c553aef50ec43448c6f825e9f41c96b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/ed1392d627434a39839fca0948b2f1b6, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/fe1942e1b7b44a53be719156b0aae2e2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/5581b46df56d4d35b3812c1ded23db32] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=131.3 K 2024-12-04T15:01:23,345 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:23,345 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/0c553aef50ec43448c6f825e9f41c96b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/ed1392d627434a39839fca0948b2f1b6, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/fe1942e1b7b44a53be719156b0aae2e2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/5581b46df56d4d35b3812c1ded23db32] 2024-12-04T15:01:23,346 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c553aef50ec43448c6f825e9f41c96b, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733324477796 2024-12-04T15:01:23,346 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed1392d627434a39839fca0948b2f1b6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733324478469 2024-12-04T15:01:23,346 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T15:01:23,346 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/B is initiating minor compaction (all files) 2024-12-04T15:01:23,346 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/B in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:23,346 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/89ab73dae04a4658b2c65b645d7b28d3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/f88c592b53be4816a42a53e8a2fde040, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/84f4e9d5f2374a14a3235c5aa0e76686, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/58b6f254ee5f49fa90a31842da574cc3] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=48.8 K 2024-12-04T15:01:23,346 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe1942e1b7b44a53be719156b0aae2e2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733324479662 2024-12-04T15:01:23,346 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 89ab73dae04a4658b2c65b645d7b28d3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733324477796 2024-12-04T15:01:23,347 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5581b46df56d4d35b3812c1ded23db32, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733324481832 2024-12-04T15:01:23,347 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting f88c592b53be4816a42a53e8a2fde040, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733324478469 2024-12-04T15:01:23,347 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 84f4e9d5f2374a14a3235c5aa0e76686, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733324479662 2024-12-04T15:01:23,347 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 58b6f254ee5f49fa90a31842da574cc3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733324481832 2024-12-04T15:01:23,355 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:23,360 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#B#compaction#541 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:23,361 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/7761ff897c0c47f3a79e9d1f76494a98 is 50, key is test_row_0/B:col10/1733324482462/Put/seqid=0 2024-12-04T15:01:23,363 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204e87a131ffc5a47d7af2648c8805a8fe3_9bc7edf6120a83a9451ae3a6357eaeb9 store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:23,365 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204e87a131ffc5a47d7af2648c8805a8fe3_9bc7edf6120a83a9451ae3a6357eaeb9, store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:23,365 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e87a131ffc5a47d7af2648c8805a8fe3_9bc7edf6120a83a9451ae3a6357eaeb9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:23,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742472_1648 (size=13153) 2024-12-04T15:01:23,375 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/7761ff897c0c47f3a79e9d1f76494a98 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/7761ff897c0c47f3a79e9d1f76494a98 2024-12-04T15:01:23,379 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/B of 9bc7edf6120a83a9451ae3a6357eaeb9 into 7761ff897c0c47f3a79e9d1f76494a98(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:23,379 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:23,380 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/B, priority=12, startTime=1733324483344; duration=0sec 2024-12-04T15:01:23,380 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:01:23,380 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:B 2024-12-04T15:01:23,380 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T15:01:23,381 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T15:01:23,381 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/C is initiating minor compaction (all files) 2024-12-04T15:01:23,381 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/C in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:23,381 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/dc04cf16fc754987a54ec44748ecd37e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/7d4b7811e7024694a33f9e34e9058cb4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/1fb911bbd50c4bd2adbeb328700b991f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/15cfd777aaac4002a450af2f0db5c355] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=48.8 K 2024-12-04T15:01:23,381 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting dc04cf16fc754987a54ec44748ecd37e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733324477796 2024-12-04T15:01:23,381 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d4b7811e7024694a33f9e34e9058cb4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733324478469 2024-12-04T15:01:23,381 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fb911bbd50c4bd2adbeb328700b991f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733324479662 2024-12-04T15:01:23,382 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 15cfd777aaac4002a450af2f0db5c355, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733324481832 2024-12-04T15:01:23,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-04T15:01:23,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:23,408 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 9bc7edf6120a83a9451ae3a6357eaeb9 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-04T15:01:23,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=A 2024-12-04T15:01:23,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:23,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=B 2024-12-04T15:01:23,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:23,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=C 2024-12-04T15:01:23,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:23,409 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#C#compaction#542 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:23,410 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/0475fca5c36949669780e828afa5aae8 is 50, key is test_row_0/C:col10/1733324482462/Put/seqid=0 2024-12-04T15:01:23,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742473_1649 (size=4469) 2024-12-04T15:01:23,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204800cec6fa7cd4917b2c4a6217d1fd9e9_9bc7edf6120a83a9451ae3a6357eaeb9 is 50, key is test_row_0/A:col10/1733324482479/Put/seqid=0 2024-12-04T15:01:23,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742474_1650 (size=13153) 2024-12-04T15:01:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-04T15:01:23,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742475_1651 (size=12454) 2024-12-04T15:01:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:23,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. as already flushing 2024-12-04T15:01:23,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324543645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:23,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:23,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324543646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:23,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:23,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:23,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324543648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:23,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:23,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324543760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:23,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324543760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:23,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324543760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:23,816 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#A#compaction#540 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:23,816 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/9d13e6cfc2724b31aae4ae8587c58709 is 175, key is test_row_0/A:col10/1733324482462/Put/seqid=0 2024-12-04T15:01:23,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742476_1652 (size=32107) 2024-12-04T15:01:23,853 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/0475fca5c36949669780e828afa5aae8 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/0475fca5c36949669780e828afa5aae8 2024-12-04T15:01:23,856 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/C of 9bc7edf6120a83a9451ae3a6357eaeb9 into 0475fca5c36949669780e828afa5aae8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:23,856 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:23,856 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/C, priority=12, startTime=1733324483344; duration=0sec 2024-12-04T15:01:23,856 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:23,856 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:C 2024-12-04T15:01:23,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:23,895 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204800cec6fa7cd4917b2c4a6217d1fd9e9_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204800cec6fa7cd4917b2c4a6217d1fd9e9_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:23,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/91b8253e87cd4a6ca95e7c6f0043854f, store: [table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:23,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/91b8253e87cd4a6ca95e7c6f0043854f is 175, key is test_row_0/A:col10/1733324482479/Put/seqid=0 2024-12-04T15:01:23,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742477_1653 (size=31255) 2024-12-04T15:01:23,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324543970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:23,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324543970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324543971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:23,973 INFO [Thread-2452 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2452. 2024-12-04T15:01:23,973 DEBUG [Thread-2452 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:01:23,973 DEBUG [Thread-2452 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:23,973 DEBUG [Thread-2452 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:23,973 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:01:23,974 INFO [Thread-2448 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2448. 2024-12-04T15:01:23,974 DEBUG [Thread-2448 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:01:23,974 DEBUG [Thread-2448 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:23,974 DEBUG [Thread-2448 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:23,974 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:01:23,975 INFO [Thread-2454 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2454. 2024-12-04T15:01:23,975 DEBUG [Thread-2454 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:01:23,975 DEBUG [Thread-2454 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:23,975 DEBUG [Thread-2454 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:23,975 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:01:23,976 INFO [Thread-2450 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2450. 2024-12-04T15:01:23,976 DEBUG [Thread-2450 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:01:23,976 DEBUG [Thread-2450 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:23,976 DEBUG [Thread-2450 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:23,976 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:01:23,976 INFO [Thread-2456 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2456. 2024-12-04T15:01:23,977 DEBUG [Thread-2456 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:01:23,977 DEBUG [Thread-2456 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:23,977 DEBUG [Thread-2456 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:23,977 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:01:24,230 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/9d13e6cfc2724b31aae4ae8587c58709 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/9d13e6cfc2724b31aae4ae8587c58709 2024-12-04T15:01:24,236 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/A of 9bc7edf6120a83a9451ae3a6357eaeb9 into 9d13e6cfc2724b31aae4ae8587c58709(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:24,236 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:24,236 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/A, priority=12, startTime=1733324483344; duration=0sec 2024-12-04T15:01:24,236 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:24,236 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:A 2024-12-04T15:01:24,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:24,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324544281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:24,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324544281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324544281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:24,305 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=394, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/91b8253e87cd4a6ca95e7c6f0043854f 2024-12-04T15:01:24,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/89e5fe4c45e74503915bbc90fc180ca5 is 50, key is test_row_0/B:col10/1733324482479/Put/seqid=0 2024-12-04T15:01:24,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742478_1654 (size=12301) 2024-12-04T15:01:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-04T15:01:24,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43180 deadline: 1733324544590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:24,591 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 9bc7edf6120a83a9451ae3a6357eaeb9 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18274 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:01:24,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:24,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:24,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:24,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43164 deadline: 1733324544640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:24,641 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,36649,1733324304801 for 'test_row_2' in 9bc7edf6120a83a9451ae3a6357eaeb9 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18328 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T15:01:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:24,722 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/89e5fe4c45e74503915bbc90fc180ca5 2024-12-04T15:01:24,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/6b0263e5c9204724a739d82e95b64433 is 50, key is test_row_0/C:col10/1733324482479/Put/seqid=0 2024-12-04T15:01:24,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742479_1655 (size=12301) 2024-12-04T15:01:24,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:24,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43128 deadline: 1733324544790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:24,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T15:01:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43140 deadline: 1733324544791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43152 deadline: 1733324544791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:24,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:24,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:24,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:24,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:24,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:24,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:24,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:01:24,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:24,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., hostname=d214edae7bd2,36649,1733324304801, seqNum=5 because the exception is null or not the one we care about 2024-12-04T15:01:25,130 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/6b0263e5c9204724a739d82e95b64433 2024-12-04T15:01:25,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/91b8253e87cd4a6ca95e7c6f0043854f as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/91b8253e87cd4a6ca95e7c6f0043854f 2024-12-04T15:01:25,139 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/91b8253e87cd4a6ca95e7c6f0043854f, entries=150, sequenceid=394, filesize=30.5 K 2024-12-04T15:01:25,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/89e5fe4c45e74503915bbc90fc180ca5 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/89e5fe4c45e74503915bbc90fc180ca5 2024-12-04T15:01:25,144 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/89e5fe4c45e74503915bbc90fc180ca5, entries=150, sequenceid=394, filesize=12.0 K 2024-12-04T15:01:25,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/6b0263e5c9204724a739d82e95b64433 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/6b0263e5c9204724a739d82e95b64433 2024-12-04T15:01:25,149 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/6b0263e5c9204724a739d82e95b64433, entries=150, sequenceid=394, filesize=12.0 K 2024-12-04T15:01:25,150 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 1742ms, sequenceid=394, compaction requested=false 2024-12-04T15:01:25,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:25,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:25,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-12-04T15:01:25,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-12-04T15:01:25,153 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-04T15:01:25,153 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8090 sec 2024-12-04T15:01:25,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.8140 sec 2024-12-04T15:01:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36649 {}] regionserver.HRegion(8855): Flush requested on 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:25,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9bc7edf6120a83a9451ae3a6357eaeb9 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-04T15:01:25,803 INFO [Thread-2443 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2443. 2024-12-04T15:01:25,804 INFO [Thread-2437 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2437. 2024-12-04T15:01:25,804 DEBUG [Thread-2443 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:01:25,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=A 2024-12-04T15:01:25,804 DEBUG [Thread-2443 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:25,804 DEBUG [Thread-2437 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:01:25,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:25,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=B 2024-12-04T15:01:25,804 DEBUG [Thread-2437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:25,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:25,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=C 2024-12-04T15:01:25,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:25,804 DEBUG [Thread-2443 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:25,804 DEBUG [Thread-2437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:25,804 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:01:25,804 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:01:25,811 INFO [Thread-2439 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2439. 2024-12-04T15:01:25,811 DEBUG [Thread-2439 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:01:25,811 DEBUG [Thread-2439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:25,811 DEBUG [Thread-2439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:25,811 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:01:25,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204560b16375c8d4a3cabc4db917ee62360_9bc7edf6120a83a9451ae3a6357eaeb9 is 50, key is test_row_0/A:col10/1733324485801/Put/seqid=0 2024-12-04T15:01:25,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742480_1656 (size=12454) 2024-12-04T15:01:26,216 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:26,225 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204560b16375c8d4a3cabc4db917ee62360_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204560b16375c8d4a3cabc4db917ee62360_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:26,228 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/d071ad961d3b4e29aa1edba75593f18c, store: [table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:26,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/d071ad961d3b4e29aa1edba75593f18c is 175, key is test_row_0/A:col10/1733324485801/Put/seqid=0 2024-12-04T15:01:26,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742481_1657 (size=31255) 2024-12-04T15:01:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-04T15:01:26,500 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:01:26,635 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=414, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/d071ad961d3b4e29aa1edba75593f18c 2024-12-04T15:01:26,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/5df1c3d79d5a4530ad20edb66fb9cf48 is 50, key is test_row_0/B:col10/1733324485801/Put/seqid=0 2024-12-04T15:01:26,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742482_1658 (size=12301) 2024-12-04T15:01:27,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/5df1c3d79d5a4530ad20edb66fb9cf48 2024-12-04T15:01:27,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/9c125d5bf11d42f8851d64b2d6c6f499 is 50, key is test_row_0/C:col10/1733324485801/Put/seqid=0 2024-12-04T15:01:27,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742483_1659 (size=12301) 2024-12-04T15:01:27,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/9c125d5bf11d42f8851d64b2d6c6f499 2024-12-04T15:01:27,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/d071ad961d3b4e29aa1edba75593f18c as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d071ad961d3b4e29aa1edba75593f18c 2024-12-04T15:01:27,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d071ad961d3b4e29aa1edba75593f18c, entries=150, sequenceid=414, filesize=30.5 K 2024-12-04T15:01:27,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/5df1c3d79d5a4530ad20edb66fb9cf48 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/5df1c3d79d5a4530ad20edb66fb9cf48 2024-12-04T15:01:27,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/5df1c3d79d5a4530ad20edb66fb9cf48, entries=150, sequenceid=414, filesize=12.0 K 2024-12-04T15:01:27,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/9c125d5bf11d42f8851d64b2d6c6f499 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/9c125d5bf11d42f8851d64b2d6c6f499 2024-12-04T15:01:27,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/9c125d5bf11d42f8851d64b2d6c6f499, entries=150, sequenceid=414, filesize=12.0 K 2024-12-04T15:01:27,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=6.71 KB/6870 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 1696ms, sequenceid=414, compaction requested=true 2024-12-04T15:01:27,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:27,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:01:27,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:27,500 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:27,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:01:27,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:27,500 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:27,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9bc7edf6120a83a9451ae3a6357eaeb9:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T15:01:27,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:01:27,500 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:27,500 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94617 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:27,500 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/B is initiating minor compaction (all files) 2024-12-04T15:01:27,500 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/A is initiating minor compaction (all files) 2024-12-04T15:01:27,501 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/B in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:27,501 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/A in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:27,501 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/7761ff897c0c47f3a79e9d1f76494a98, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/89e5fe4c45e74503915bbc90fc180ca5, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/5df1c3d79d5a4530ad20edb66fb9cf48] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=36.9 K 2024-12-04T15:01:27,501 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/9d13e6cfc2724b31aae4ae8587c58709, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/91b8253e87cd4a6ca95e7c6f0043854f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d071ad961d3b4e29aa1edba75593f18c] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=92.4 K 2024-12-04T15:01:27,501 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:27,501 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. files: [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/9d13e6cfc2724b31aae4ae8587c58709, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/91b8253e87cd4a6ca95e7c6f0043854f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d071ad961d3b4e29aa1edba75593f18c] 2024-12-04T15:01:27,501 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7761ff897c0c47f3a79e9d1f76494a98, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733324481832 2024-12-04T15:01:27,501 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d13e6cfc2724b31aae4ae8587c58709, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733324481832 2024-12-04T15:01:27,501 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 89e5fe4c45e74503915bbc90fc180ca5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733324482474 2024-12-04T15:01:27,501 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91b8253e87cd4a6ca95e7c6f0043854f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733324482474 2024-12-04T15:01:27,502 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 5df1c3d79d5a4530ad20edb66fb9cf48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733324483647 2024-12-04T15:01:27,502 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] compactions.Compactor(225): Compacting d071ad961d3b4e29aa1edba75593f18c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733324483647 2024-12-04T15:01:27,509 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#B#compaction#549 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:27,509 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/06131feb523641aa8369ba73cd760391 is 50, key is test_row_0/B:col10/1733324485801/Put/seqid=0 2024-12-04T15:01:27,510 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:27,514 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412046bafcf79d06c4d56a074e736af24c6df_9bc7edf6120a83a9451ae3a6357eaeb9 store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:27,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742484_1660 (size=13255) 2024-12-04T15:01:27,516 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412046bafcf79d06c4d56a074e736af24c6df_9bc7edf6120a83a9451ae3a6357eaeb9, store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:27,517 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412046bafcf79d06c4d56a074e736af24c6df_9bc7edf6120a83a9451ae3a6357eaeb9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:27,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742485_1661 (size=4469) 2024-12-04T15:01:27,923 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#A#compaction#550 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:27,924 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/d0615b9f52a54920b04a64e6af7df703 is 175, key is test_row_0/A:col10/1733324485801/Put/seqid=0 2024-12-04T15:01:27,926 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/06131feb523641aa8369ba73cd760391 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/06131feb523641aa8369ba73cd760391 2024-12-04T15:01:27,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742486_1662 (size=32209) 2024-12-04T15:01:27,931 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/B of 9bc7edf6120a83a9451ae3a6357eaeb9 into 06131feb523641aa8369ba73cd760391(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:27,931 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:27,931 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/B, priority=13, startTime=1733324487500; duration=0sec 2024-12-04T15:01:27,931 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:01:27,931 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:B 2024-12-04T15:01:27,931 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:01:27,932 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:01:27,932 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1541): 9bc7edf6120a83a9451ae3a6357eaeb9/C is initiating minor compaction (all files) 2024-12-04T15:01:27,932 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9bc7edf6120a83a9451ae3a6357eaeb9/C in TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:27,933 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/0475fca5c36949669780e828afa5aae8, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/6b0263e5c9204724a739d82e95b64433, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/9c125d5bf11d42f8851d64b2d6c6f499] into tmpdir=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp, totalSize=36.9 K 2024-12-04T15:01:27,933 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 0475fca5c36949669780e828afa5aae8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733324481832 2024-12-04T15:01:27,933 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b0263e5c9204724a739d82e95b64433, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733324482474 2024-12-04T15:01:27,933 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c125d5bf11d42f8851d64b2d6c6f499, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733324483647 2024-12-04T15:01:27,940 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9bc7edf6120a83a9451ae3a6357eaeb9#C#compaction#551 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:01:27,940 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/4498517c9e284caeb9a5bfee23925cdc is 50, key is test_row_0/C:col10/1733324485801/Put/seqid=0 2024-12-04T15:01:27,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742487_1663 (size=13255) 2024-12-04T15:01:28,338 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/d0615b9f52a54920b04a64e6af7df703 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d0615b9f52a54920b04a64e6af7df703 2024-12-04T15:01:28,345 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/A of 9bc7edf6120a83a9451ae3a6357eaeb9 into d0615b9f52a54920b04a64e6af7df703(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:28,345 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:28,345 INFO [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/A, priority=13, startTime=1733324487499; duration=0sec 2024-12-04T15:01:28,345 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:28,345 DEBUG [RS:0;d214edae7bd2:36649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:A 2024-12-04T15:01:28,348 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/4498517c9e284caeb9a5bfee23925cdc as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/4498517c9e284caeb9a5bfee23925cdc 2024-12-04T15:01:28,353 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9bc7edf6120a83a9451ae3a6357eaeb9/C of 9bc7edf6120a83a9451ae3a6357eaeb9 into 4498517c9e284caeb9a5bfee23925cdc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:01:28,353 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9bc7edf6120a83a9451ae3a6357eaeb9: 2024-12-04T15:01:28,353 INFO [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., storeName=9bc7edf6120a83a9451ae3a6357eaeb9/C, priority=13, startTime=1733324487500; duration=0sec 2024-12-04T15:01:28,353 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:01:28,353 DEBUG [RS:0;d214edae7bd2:36649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9bc7edf6120a83a9451ae3a6357eaeb9:C 2024-12-04T15:01:34,672 INFO [Thread-2441 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2441. 2024-12-04T15:01:34,672 DEBUG [Thread-2441 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:01:34,672 DEBUG [Thread-2441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:34,673 DEBUG [Thread-2441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:34,673 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:01:34,692 INFO [Thread-2445 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2445. 2024-12-04T15:01:34,692 DEBUG [Thread-2445 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T15:01:34,692 DEBUG [Thread-2445 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:34,692 DEBUG [Thread-2445 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:34,692 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:01:34,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T15:01:34,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-12-04T15:01:34,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-12-04T15:01:34,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-04T15:01:34,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-12-04T15:01:34,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-04T15:01:34,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T15:01:34,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4066 2024-12-04T15:01:34,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3880 2024-12-04T15:01:34,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3875 2024-12-04T15:01:34,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4064 2024-12-04T15:01:34,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3889 2024-12-04T15:01:34,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T15:01:34,693 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T15:01:34,693 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T15:01:34,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:34,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:34,693 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:01:34,694 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T15:01:34,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T15:01:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-04T15:01:34,697 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324494696"}]},"ts":"1733324494696"} 2024-12-04T15:01:34,698 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T15:01:34,698 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T15:01:34,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T15:01:34,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9bc7edf6120a83a9451ae3a6357eaeb9, UNASSIGN}] 2024-12-04T15:01:34,701 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9bc7edf6120a83a9451ae3a6357eaeb9, UNASSIGN 2024-12-04T15:01:34,702 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=9bc7edf6120a83a9451ae3a6357eaeb9, regionState=CLOSING, regionLocation=d214edae7bd2,36649,1733324304801 2024-12-04T15:01:34,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9bc7edf6120a83a9451ae3a6357eaeb9, UNASSIGN because future has completed 2024-12-04T15:01:34,703 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T15:01:34,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801}] 2024-12-04T15:01:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-04T15:01:34,856 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(122): Close 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:34,856 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T15:01:34,856 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1722): Closing 9bc7edf6120a83a9451ae3a6357eaeb9, disabling compactions & flushes 2024-12-04T15:01:34,856 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:34,856 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:34,856 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. after waiting 0 ms 2024-12-04T15:01:34,856 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:34,856 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(2902): Flushing 9bc7edf6120a83a9451ae3a6357eaeb9 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-04T15:01:34,856 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=A 2024-12-04T15:01:34,857 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:34,857 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=B 2024-12-04T15:01:34,857 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:34,857 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9bc7edf6120a83a9451ae3a6357eaeb9, store=C 2024-12-04T15:01:34,857 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T15:01:34,861 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120459fde5c3bd10419c9a733446e7d3f026_9bc7edf6120a83a9451ae3a6357eaeb9 is 50, key is test_row_1/A:col10/1733324485810/Put/seqid=0 2024-12-04T15:01:34,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742488_1664 (size=9914) 2024-12-04T15:01:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-04T15:01:35,265 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:01:35,268 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120459fde5c3bd10419c9a733446e7d3f026_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120459fde5c3bd10419c9a733446e7d3f026_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:35,269 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/fd8988c3affa4b1e8bd828504c4655f1, store: [table=TestAcidGuarantees family=A region=9bc7edf6120a83a9451ae3a6357eaeb9] 2024-12-04T15:01:35,269 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/fd8988c3affa4b1e8bd828504c4655f1 is 175, key is test_row_1/A:col10/1733324485810/Put/seqid=0 2024-12-04T15:01:35,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742489_1665 (size=22561) 2024-12-04T15:01:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-04T15:01:35,673 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=423, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/fd8988c3affa4b1e8bd828504c4655f1 2024-12-04T15:01:35,683 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/66abd6172aa14ad8a0ff4f8358065c36 is 50, key is test_row_1/B:col10/1733324485810/Put/seqid=0 2024-12-04T15:01:35,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742490_1666 (size=9857) 2024-12-04T15:01:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-04T15:01:36,088 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/66abd6172aa14ad8a0ff4f8358065c36 2024-12-04T15:01:36,094 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/334886a13a4d4ff998e1b95709450c89 is 50, key is test_row_1/C:col10/1733324485810/Put/seqid=0 2024-12-04T15:01:36,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742491_1667 (size=9857) 2024-12-04T15:01:36,499 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/334886a13a4d4ff998e1b95709450c89 2024-12-04T15:01:36,505 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/A/fd8988c3affa4b1e8bd828504c4655f1 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/fd8988c3affa4b1e8bd828504c4655f1 2024-12-04T15:01:36,508 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/fd8988c3affa4b1e8bd828504c4655f1, entries=100, sequenceid=423, filesize=22.0 K 2024-12-04T15:01:36,509 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/B/66abd6172aa14ad8a0ff4f8358065c36 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/66abd6172aa14ad8a0ff4f8358065c36 2024-12-04T15:01:36,512 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/66abd6172aa14ad8a0ff4f8358065c36, entries=100, sequenceid=423, filesize=9.6 K 2024-12-04T15:01:36,513 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/.tmp/C/334886a13a4d4ff998e1b95709450c89 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/334886a13a4d4ff998e1b95709450c89 2024-12-04T15:01:36,517 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/334886a13a4d4ff998e1b95709450c89, entries=100, sequenceid=423, filesize=9.6 K 2024-12-04T15:01:36,517 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 1661ms, sequenceid=423, compaction requested=false 2024-12-04T15:01:36,518 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/1a9f523feb5f43b29503e27a03b916ff, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/ae6ba2a183a144f2af3b0738c699b3f7, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/bcebc65e7d7a4b05aaacdd4a4e44c5c7, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d8710a2b94e346a5a7198b4703ea849e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/f67adf1c36794312b504fbf77365c58a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/a9ecfd1e5b0845cd9097c954fdaca840, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/01d6642efb1049ab97c0ae728318364b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/0d88fc2dcb084d8483d854adbc29b6ae, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/242a98046a13415b8cbde9af10ae5db2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/203490922aca487dbf0e3f563f770e98, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/da89b10e39904455850ba781940da40d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/30c67df7db814eadaa62184c9ebdce22, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/b41afaddb4c24c1eac0f9f51783d8a38, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/331e0c19fba44a93871b09fb1e2a813f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/437a648d11514b11a105c769ccb1dea0, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/b3216f2b5e884bd0ba7ae1aca4eac9fc, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/f7987f58f98c4ee7ba99f08291861ec2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d6197ed4d5a14e2c8bb1a0dd245db1c2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/e2546e7b4f6b4d9e87509633152b85e3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/a37e684930d9462db691a6d662e39347, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/bad394ceb43045ec8624a528091ab1d1, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/0c553aef50ec43448c6f825e9f41c96b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/ed1392d627434a39839fca0948b2f1b6, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/fe1942e1b7b44a53be719156b0aae2e2, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/5581b46df56d4d35b3812c1ded23db32, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/9d13e6cfc2724b31aae4ae8587c58709, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/91b8253e87cd4a6ca95e7c6f0043854f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d071ad961d3b4e29aa1edba75593f18c] to archive 2024-12-04T15:01:36,519 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T15:01:36,520 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/1a9f523feb5f43b29503e27a03b916ff to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/1a9f523feb5f43b29503e27a03b916ff 2024-12-04T15:01:36,521 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/ae6ba2a183a144f2af3b0738c699b3f7 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/ae6ba2a183a144f2af3b0738c699b3f7 2024-12-04T15:01:36,522 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/bcebc65e7d7a4b05aaacdd4a4e44c5c7 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/bcebc65e7d7a4b05aaacdd4a4e44c5c7 2024-12-04T15:01:36,523 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d8710a2b94e346a5a7198b4703ea849e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d8710a2b94e346a5a7198b4703ea849e 2024-12-04T15:01:36,524 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/f67adf1c36794312b504fbf77365c58a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/f67adf1c36794312b504fbf77365c58a 2024-12-04T15:01:36,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/a9ecfd1e5b0845cd9097c954fdaca840 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/a9ecfd1e5b0845cd9097c954fdaca840 2024-12-04T15:01:36,527 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/01d6642efb1049ab97c0ae728318364b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/01d6642efb1049ab97c0ae728318364b 2024-12-04T15:01:36,529 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/0d88fc2dcb084d8483d854adbc29b6ae to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/0d88fc2dcb084d8483d854adbc29b6ae 2024-12-04T15:01:36,534 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/242a98046a13415b8cbde9af10ae5db2 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/242a98046a13415b8cbde9af10ae5db2 2024-12-04T15:01:36,535 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/203490922aca487dbf0e3f563f770e98 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/203490922aca487dbf0e3f563f770e98 2024-12-04T15:01:36,535 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/da89b10e39904455850ba781940da40d to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/da89b10e39904455850ba781940da40d 2024-12-04T15:01:36,536 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/30c67df7db814eadaa62184c9ebdce22 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/30c67df7db814eadaa62184c9ebdce22 2024-12-04T15:01:36,537 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/b41afaddb4c24c1eac0f9f51783d8a38 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/b41afaddb4c24c1eac0f9f51783d8a38 2024-12-04T15:01:36,538 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/331e0c19fba44a93871b09fb1e2a813f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/331e0c19fba44a93871b09fb1e2a813f 2024-12-04T15:01:36,539 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/437a648d11514b11a105c769ccb1dea0 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/437a648d11514b11a105c769ccb1dea0 2024-12-04T15:01:36,539 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/b3216f2b5e884bd0ba7ae1aca4eac9fc to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/b3216f2b5e884bd0ba7ae1aca4eac9fc 2024-12-04T15:01:36,540 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/f7987f58f98c4ee7ba99f08291861ec2 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/f7987f58f98c4ee7ba99f08291861ec2 2024-12-04T15:01:36,541 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d6197ed4d5a14e2c8bb1a0dd245db1c2 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d6197ed4d5a14e2c8bb1a0dd245db1c2 2024-12-04T15:01:36,542 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/e2546e7b4f6b4d9e87509633152b85e3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/e2546e7b4f6b4d9e87509633152b85e3 2024-12-04T15:01:36,542 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/a37e684930d9462db691a6d662e39347 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/a37e684930d9462db691a6d662e39347 2024-12-04T15:01:36,543 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/bad394ceb43045ec8624a528091ab1d1 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/bad394ceb43045ec8624a528091ab1d1 2024-12-04T15:01:36,544 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/0c553aef50ec43448c6f825e9f41c96b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/0c553aef50ec43448c6f825e9f41c96b 2024-12-04T15:01:36,544 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/ed1392d627434a39839fca0948b2f1b6 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/ed1392d627434a39839fca0948b2f1b6 2024-12-04T15:01:36,545 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/fe1942e1b7b44a53be719156b0aae2e2 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/fe1942e1b7b44a53be719156b0aae2e2 2024-12-04T15:01:36,546 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/5581b46df56d4d35b3812c1ded23db32 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/5581b46df56d4d35b3812c1ded23db32 2024-12-04T15:01:36,547 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/9d13e6cfc2724b31aae4ae8587c58709 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/9d13e6cfc2724b31aae4ae8587c58709 2024-12-04T15:01:36,547 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/91b8253e87cd4a6ca95e7c6f0043854f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/91b8253e87cd4a6ca95e7c6f0043854f 2024-12-04T15:01:36,548 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d071ad961d3b4e29aa1edba75593f18c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d071ad961d3b4e29aa1edba75593f18c 2024-12-04T15:01:36,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/c78c91649e824390a14fd1ad8c23b90e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/ae407c74e7714c1b9633735926c231e6, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/f372fbc0b0c3411fb1cfcd9f8d20e5b3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/35ac3d6b9940478cab76df6a77142649, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/528160467848429a8a9151ec6607b25d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/d3a7bdd75df44aac83663fc6a5896483, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/740b364c50cd4cff8d1de379beec8515, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/b47990a598644af0a22871def6575f1b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/835107bab2f84af8ac5bc368936de601, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/92b3c753f5b6404f86bceff1815263db, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/79b12ab3ab33468abea445e81007b833, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/38c6969167024e859850f017e20a995e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/d4bc9fa710074e61bca64acf7ff29050, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/93807d414475485e896e58f17ebacde4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/ea0e54d4e5b6404a89d8fc8f281d19cf, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/d41d33afb56e444ab2c2cc8993dc6391, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/b4b958945bcc47308cfdd89fb694c015, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/7d7ba7aea7144034a705af1b6c84c08c, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/1bd9ce90782d4476944df36790b5f082, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/def460423ccc4b54ba7c7482e47be509, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/89ab73dae04a4658b2c65b645d7b28d3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/0099da0038a743b4bb6b7feeab932f91, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/f88c592b53be4816a42a53e8a2fde040, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/84f4e9d5f2374a14a3235c5aa0e76686, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/7761ff897c0c47f3a79e9d1f76494a98, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/58b6f254ee5f49fa90a31842da574cc3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/89e5fe4c45e74503915bbc90fc180ca5, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/5df1c3d79d5a4530ad20edb66fb9cf48] to archive 2024-12-04T15:01:36,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T15:01:36,552 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/c78c91649e824390a14fd1ad8c23b90e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/c78c91649e824390a14fd1ad8c23b90e 2024-12-04T15:01:36,553 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/ae407c74e7714c1b9633735926c231e6 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/ae407c74e7714c1b9633735926c231e6 2024-12-04T15:01:36,553 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/f372fbc0b0c3411fb1cfcd9f8d20e5b3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/f372fbc0b0c3411fb1cfcd9f8d20e5b3 2024-12-04T15:01:36,554 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/35ac3d6b9940478cab76df6a77142649 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/35ac3d6b9940478cab76df6a77142649 2024-12-04T15:01:36,555 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/528160467848429a8a9151ec6607b25d to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/528160467848429a8a9151ec6607b25d 2024-12-04T15:01:36,556 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/d3a7bdd75df44aac83663fc6a5896483 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/d3a7bdd75df44aac83663fc6a5896483 2024-12-04T15:01:36,557 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/740b364c50cd4cff8d1de379beec8515 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/740b364c50cd4cff8d1de379beec8515 2024-12-04T15:01:36,558 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/b47990a598644af0a22871def6575f1b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/b47990a598644af0a22871def6575f1b 2024-12-04T15:01:36,561 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/835107bab2f84af8ac5bc368936de601 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/835107bab2f84af8ac5bc368936de601 2024-12-04T15:01:36,568 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/92b3c753f5b6404f86bceff1815263db to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/92b3c753f5b6404f86bceff1815263db 2024-12-04T15:01:36,570 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/79b12ab3ab33468abea445e81007b833 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/79b12ab3ab33468abea445e81007b833 2024-12-04T15:01:36,571 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/38c6969167024e859850f017e20a995e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/38c6969167024e859850f017e20a995e 2024-12-04T15:01:36,572 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/d4bc9fa710074e61bca64acf7ff29050 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/d4bc9fa710074e61bca64acf7ff29050 2024-12-04T15:01:36,573 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/93807d414475485e896e58f17ebacde4 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/93807d414475485e896e58f17ebacde4 2024-12-04T15:01:36,574 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/ea0e54d4e5b6404a89d8fc8f281d19cf to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/ea0e54d4e5b6404a89d8fc8f281d19cf 2024-12-04T15:01:36,575 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/d41d33afb56e444ab2c2cc8993dc6391 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/d41d33afb56e444ab2c2cc8993dc6391 2024-12-04T15:01:36,576 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/b4b958945bcc47308cfdd89fb694c015 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/b4b958945bcc47308cfdd89fb694c015 2024-12-04T15:01:36,577 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/7d7ba7aea7144034a705af1b6c84c08c to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/7d7ba7aea7144034a705af1b6c84c08c 2024-12-04T15:01:36,578 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/1bd9ce90782d4476944df36790b5f082 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/1bd9ce90782d4476944df36790b5f082 2024-12-04T15:01:36,579 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/def460423ccc4b54ba7c7482e47be509 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/def460423ccc4b54ba7c7482e47be509 2024-12-04T15:01:36,580 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/89ab73dae04a4658b2c65b645d7b28d3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/89ab73dae04a4658b2c65b645d7b28d3 2024-12-04T15:01:36,580 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/0099da0038a743b4bb6b7feeab932f91 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/0099da0038a743b4bb6b7feeab932f91 2024-12-04T15:01:36,581 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/f88c592b53be4816a42a53e8a2fde040 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/f88c592b53be4816a42a53e8a2fde040 2024-12-04T15:01:36,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/84f4e9d5f2374a14a3235c5aa0e76686 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/84f4e9d5f2374a14a3235c5aa0e76686 2024-12-04T15:01:36,583 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/7761ff897c0c47f3a79e9d1f76494a98 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/7761ff897c0c47f3a79e9d1f76494a98 2024-12-04T15:01:36,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/58b6f254ee5f49fa90a31842da574cc3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/58b6f254ee5f49fa90a31842da574cc3 2024-12-04T15:01:36,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/89e5fe4c45e74503915bbc90fc180ca5 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/89e5fe4c45e74503915bbc90fc180ca5 2024-12-04T15:01:36,585 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/5df1c3d79d5a4530ad20edb66fb9cf48 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/5df1c3d79d5a4530ad20edb66fb9cf48 2024-12-04T15:01:36,587 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/eaa4632ab1e14ff9aa75cc41ce21664d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/532e380a552b48ce8bae40a3f5f4bff5, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/1e4ef57e37ad4fd5a340a9d15194b09e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/6bf8bebc02a34eeabd4b99c7cfe7bcf1, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/ee5ad04e625b4397ae3423cb0e0bb92a, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/191db78b85e84a5ba8bf595506309c36, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/e1521509de1845e9953c922772f47480, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/db4ecf1c36ac4b70b1879f5579849dcb, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/c6a2d669e1ed4923b413171488bfc45e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/f127d2edfb144927bfac25a031b0ebf3, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/19c1f1a89a9c47fa8f4f41f038d54cea, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/e2c16828bc8b4c6b9d8030da37fe4848, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/5b1c17c65b9549c683fdb468ff72c7ce, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/0154d4363b504819bc7caac30b480c5d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/6ed2ed9c44ea4f96bf9139d7506c7588, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/91382da585324870ae2e54dccb0e4f8d, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/e5e7169813904021bd31ca972739366b, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/bd05cc08962844f0b711fc2dffc34da4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/73c1c5ec9ee540879a61c2ec22b08964, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/fb39dd9cc1c7463b85028db289638453, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/dc04cf16fc754987a54ec44748ecd37e, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/f508b9b02aba41ecaac1e32dac234670, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/7d4b7811e7024694a33f9e34e9058cb4, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/1fb911bbd50c4bd2adbeb328700b991f, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/0475fca5c36949669780e828afa5aae8, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/15cfd777aaac4002a450af2f0db5c355, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/6b0263e5c9204724a739d82e95b64433, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/9c125d5bf11d42f8851d64b2d6c6f499] to archive 2024-12-04T15:01:36,587 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T15:01:36,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/eaa4632ab1e14ff9aa75cc41ce21664d to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/eaa4632ab1e14ff9aa75cc41ce21664d 2024-12-04T15:01:36,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/532e380a552b48ce8bae40a3f5f4bff5 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/532e380a552b48ce8bae40a3f5f4bff5 2024-12-04T15:01:36,590 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/1e4ef57e37ad4fd5a340a9d15194b09e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/1e4ef57e37ad4fd5a340a9d15194b09e 2024-12-04T15:01:36,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/6bf8bebc02a34eeabd4b99c7cfe7bcf1 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/6bf8bebc02a34eeabd4b99c7cfe7bcf1 2024-12-04T15:01:36,593 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/ee5ad04e625b4397ae3423cb0e0bb92a to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/ee5ad04e625b4397ae3423cb0e0bb92a 2024-12-04T15:01:36,594 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/191db78b85e84a5ba8bf595506309c36 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/191db78b85e84a5ba8bf595506309c36 2024-12-04T15:01:36,595 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/e1521509de1845e9953c922772f47480 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/e1521509de1845e9953c922772f47480 2024-12-04T15:01:36,596 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/db4ecf1c36ac4b70b1879f5579849dcb to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/db4ecf1c36ac4b70b1879f5579849dcb 2024-12-04T15:01:36,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/c6a2d669e1ed4923b413171488bfc45e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/c6a2d669e1ed4923b413171488bfc45e 2024-12-04T15:01:36,604 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/f127d2edfb144927bfac25a031b0ebf3 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/f127d2edfb144927bfac25a031b0ebf3 2024-12-04T15:01:36,605 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/19c1f1a89a9c47fa8f4f41f038d54cea to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/19c1f1a89a9c47fa8f4f41f038d54cea 2024-12-04T15:01:36,606 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/e2c16828bc8b4c6b9d8030da37fe4848 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/e2c16828bc8b4c6b9d8030da37fe4848 2024-12-04T15:01:36,607 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/5b1c17c65b9549c683fdb468ff72c7ce to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/5b1c17c65b9549c683fdb468ff72c7ce 2024-12-04T15:01:36,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/0154d4363b504819bc7caac30b480c5d to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/0154d4363b504819bc7caac30b480c5d 2024-12-04T15:01:36,609 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/6ed2ed9c44ea4f96bf9139d7506c7588 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/6ed2ed9c44ea4f96bf9139d7506c7588 2024-12-04T15:01:36,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/91382da585324870ae2e54dccb0e4f8d to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/91382da585324870ae2e54dccb0e4f8d 2024-12-04T15:01:36,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/e5e7169813904021bd31ca972739366b to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/e5e7169813904021bd31ca972739366b 2024-12-04T15:01:36,611 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/bd05cc08962844f0b711fc2dffc34da4 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/bd05cc08962844f0b711fc2dffc34da4 2024-12-04T15:01:36,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/73c1c5ec9ee540879a61c2ec22b08964 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/73c1c5ec9ee540879a61c2ec22b08964 2024-12-04T15:01:36,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/fb39dd9cc1c7463b85028db289638453 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/fb39dd9cc1c7463b85028db289638453 2024-12-04T15:01:36,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/dc04cf16fc754987a54ec44748ecd37e to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/dc04cf16fc754987a54ec44748ecd37e 2024-12-04T15:01:36,614 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/f508b9b02aba41ecaac1e32dac234670 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/f508b9b02aba41ecaac1e32dac234670 2024-12-04T15:01:36,615 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/7d4b7811e7024694a33f9e34e9058cb4 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/7d4b7811e7024694a33f9e34e9058cb4 2024-12-04T15:01:36,615 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/1fb911bbd50c4bd2adbeb328700b991f to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/1fb911bbd50c4bd2adbeb328700b991f 2024-12-04T15:01:36,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/0475fca5c36949669780e828afa5aae8 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/0475fca5c36949669780e828afa5aae8 2024-12-04T15:01:36,617 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/15cfd777aaac4002a450af2f0db5c355 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/15cfd777aaac4002a450af2f0db5c355 2024-12-04T15:01:36,617 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/6b0263e5c9204724a739d82e95b64433 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/6b0263e5c9204724a739d82e95b64433 2024-12-04T15:01:36,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/9c125d5bf11d42f8851d64b2d6c6f499 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/9c125d5bf11d42f8851d64b2d6c6f499 2024-12-04T15:01:36,621 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/recovered.edits/426.seqid, newMaxSeqId=426, maxSeqId=4 2024-12-04T15:01:36,622 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. 2024-12-04T15:01:36,622 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1676): Region close journal for 9bc7edf6120a83a9451ae3a6357eaeb9: Waiting for close lock at 1733324494856Running coprocessor pre-close hooks at 1733324494856Disabling compacts and flushes for region at 1733324494856Disabling writes for close at 1733324494856Obtaining lock to block concurrent updates at 1733324494856Preparing flush snapshotting stores in 9bc7edf6120a83a9451ae3a6357eaeb9 at 1733324494856Finished memstore snapshotting TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733324494857 (+1 ms)Flushing stores of TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9. at 1733324494857Flushing 9bc7edf6120a83a9451ae3a6357eaeb9/A: creating writer at 1733324494857Flushing mob file 9bc7edf6120a83a9451ae3a6357eaeb9/A: appending metadata at 1733324494861 (+4 ms)Flushing mob file 9bc7edf6120a83a9451ae3a6357eaeb9/A: closing flushed file at 1733324494861Flushing 9bc7edf6120a83a9451ae3a6357eaeb9/A: appending metadata at 1733324495269 (+408 ms)Flushing 9bc7edf6120a83a9451ae3a6357eaeb9/A: closing flushed file at 1733324495269Flushing 9bc7edf6120a83a9451ae3a6357eaeb9/B: creating writer at 1733324495679 (+410 ms)Flushing 9bc7edf6120a83a9451ae3a6357eaeb9/B: appending metadata at 1733324495682 (+3 ms)Flushing 9bc7edf6120a83a9451ae3a6357eaeb9/B: closing flushed file at 1733324495682Flushing 9bc7edf6120a83a9451ae3a6357eaeb9/C: creating writer at 1733324496091 (+409 ms)Flushing 9bc7edf6120a83a9451ae3a6357eaeb9/C: appending metadata at 1733324496093 (+2 ms)Flushing 9bc7edf6120a83a9451ae3a6357eaeb9/C: closing flushed file at 1733324496093Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5471ed6a: reopening flushed file at 1733324496505 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42e315b9: reopening flushed file at 1733324496508 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6749b421: reopening flushed file at 1733324496512 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 9bc7edf6120a83a9451ae3a6357eaeb9 in 1661ms, sequenceid=423, compaction requested=false at 1733324496517 (+5 ms)Writing region close event to WAL at 1733324496619 (+102 ms)Running coprocessor post-close hooks at 1733324496622 (+3 ms)Closed at 1733324496622 2024-12-04T15:01:36,624 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(157): Closed 9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,624 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=9bc7edf6120a83a9451ae3a6357eaeb9, regionState=CLOSED 2024-12-04T15:01:36,625 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 because future has completed 2024-12-04T15:01:36,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-12-04T15:01:36,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseRegionProcedure 9bc7edf6120a83a9451ae3a6357eaeb9, server=d214edae7bd2,36649,1733324304801 in 1.9230 sec 2024-12-04T15:01:36,628 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-12-04T15:01:36,629 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9bc7edf6120a83a9451ae3a6357eaeb9, UNASSIGN in 1.9270 sec 2024-12-04T15:01:36,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-04T15:01:36,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9300 sec 2024-12-04T15:01:36,631 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324496631"}]},"ts":"1733324496631"} 2024-12-04T15:01:36,632 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T15:01:36,632 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T15:01:36,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9390 sec 2024-12-04T15:01:36,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-04T15:01:36,840 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:01:36,841 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T15:01:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:01:36,842 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:01:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-04T15:01:36,843 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=170, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:01:36,845 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,847 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A, FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B, FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C, FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/recovered.edits] 2024-12-04T15:01:36,849 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d0615b9f52a54920b04a64e6af7df703 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/d0615b9f52a54920b04a64e6af7df703 2024-12-04T15:01:36,850 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/fd8988c3affa4b1e8bd828504c4655f1 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/A/fd8988c3affa4b1e8bd828504c4655f1 2024-12-04T15:01:36,852 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/06131feb523641aa8369ba73cd760391 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/06131feb523641aa8369ba73cd760391 2024-12-04T15:01:36,853 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/66abd6172aa14ad8a0ff4f8358065c36 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/B/66abd6172aa14ad8a0ff4f8358065c36 2024-12-04T15:01:36,855 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/334886a13a4d4ff998e1b95709450c89 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/334886a13a4d4ff998e1b95709450c89 2024-12-04T15:01:36,856 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/4498517c9e284caeb9a5bfee23925cdc to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/C/4498517c9e284caeb9a5bfee23925cdc 2024-12-04T15:01:36,858 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/recovered.edits/426.seqid to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9/recovered.edits/426.seqid 2024-12-04T15:01:36,859 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/default/TestAcidGuarantees/9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,859 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T15:01:36,859 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T15:01:36,860 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-04T15:01:36,862 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120407d15ad13f2847019e021d783c4c02ab_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120407d15ad13f2847019e021d783c4c02ab_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,863 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120413ce2c73b382465684ae73b5b7f5bdfd_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120413ce2c73b382465684ae73b5b7f5bdfd_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,864 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120422e489b6e37140fba6b4cbeb29cb5f1c_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120422e489b6e37140fba6b4cbeb29cb5f1c_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,864 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120439e711b3909140868f3ddcd5de5c8076_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120439e711b3909140868f3ddcd5de5c8076_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,865 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120442f56d6ebbdb4285a6cfaa2187fa3980_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120442f56d6ebbdb4285a6cfaa2187fa3980_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,866 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120446eaba3856344d4bb8c674ede2c9a7b1_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120446eaba3856344d4bb8c674ede2c9a7b1_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,867 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204560b16375c8d4a3cabc4db917ee62360_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204560b16375c8d4a3cabc4db917ee62360_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,868 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120459fde5c3bd10419c9a733446e7d3f026_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120459fde5c3bd10419c9a733446e7d3f026_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,868 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204800cec6fa7cd4917b2c4a6217d1fd9e9_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204800cec6fa7cd4917b2c4a6217d1fd9e9_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,869 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048662564f24b04c3f884d4b8f804ffd10_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048662564f24b04c3f884d4b8f804ffd10_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,870 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120499bafa0c989143c9ba79b7d892ebf53b_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120499bafa0c989143c9ba79b7d892ebf53b_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,871 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412049d285a92481a48bebd160fe76932df61_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412049d285a92481a48bebd160fe76932df61_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,872 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a6c6077dde674ad3b7134e034f7e5b73_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a6c6077dde674ad3b7134e034f7e5b73_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,873 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ade3813fe9f542e7b737b5c214d36be3_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ade3813fe9f542e7b737b5c214d36be3_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,874 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204afe73506933e43ad8b9a338b96436c87_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204afe73506933e43ad8b9a338b96436c87_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,875 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c7b20f55ece54d4685c6a2f0db8f7c81_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c7b20f55ece54d4685c6a2f0db8f7c81_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,876 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c7f1c968286e49aaa352dbe61bf4dec6_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c7f1c968286e49aaa352dbe61bf4dec6_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,876 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d79bdc0119274369b619f7f00c3a9dfa_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d79bdc0119274369b619f7f00c3a9dfa_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,877 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204dfd0135866d2460d8b59d311d1a13984_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204dfd0135866d2460d8b59d311d1a13984_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,878 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e4f4dc2e4f934e7cae713bd6af8295bf_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e4f4dc2e4f934e7cae713bd6af8295bf_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,879 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f4a39ff6436f4df4a4d42301d776b4b3_9bc7edf6120a83a9451ae3a6357eaeb9 to hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f4a39ff6436f4df4a4d42301d776b4b3_9bc7edf6120a83a9451ae3a6357eaeb9 2024-12-04T15:01:36,880 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T15:01:36,882 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=170, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:01:36,884 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T15:01:36,886 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T15:01:36,888 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=170, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:01:36,888 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T15:01:36,888 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733324496888"}]},"ts":"9223372036854775807"} 2024-12-04T15:01:36,890 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T15:01:36,890 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9bc7edf6120a83a9451ae3a6357eaeb9, NAME => 'TestAcidGuarantees,,1733324461857.9bc7edf6120a83a9451ae3a6357eaeb9.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T15:01:36,890 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T15:01:36,890 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733324496890"}]},"ts":"9223372036854775807"} 2024-12-04T15:01:36,893 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T15:01:36,893 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=170, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T15:01:36,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 52 msec 2024-12-04T15:01:36,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-04T15:01:36,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T15:01:36,951 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T15:01:36,959 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 250), OpenFileDescriptor=451 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=490 (was 440) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9371 (was 9558) 2024-12-04T15:01:36,959 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T15:01:36,959 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T15:01:36,959 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T15:01:36,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:36,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:36,959 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:01:36,959 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T15:01:36,960 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=855483586, stopped=false 2024-12-04T15:01:36,960 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d214edae7bd2,39449,1733324304115 2024-12-04T15:01:37,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T15:01:37,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36649-0x101a2238e4c0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T15:01:37,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T15:01:37,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36649-0x101a2238e4c0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T15:01:37,051 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T15:01:37,053 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36649-0x101a2238e4c0001, quorum=127.0.0.1:52559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T15:01:37,053 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T15:01:37,053 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T15:01:37,053 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T15:01:37,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:37,054 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd214edae7bd2,36649,1733324304801' ***** 2024-12-04T15:01:37,055 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T15:01:37,055 INFO [RS:0;d214edae7bd2:36649 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T15:01:37,056 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T15:01:37,056 INFO [RS:0;d214edae7bd2:36649 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T15:01:37,057 INFO [RS:0;d214edae7bd2:36649 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T15:01:37,057 INFO [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(959): stopping server d214edae7bd2,36649,1733324304801 2024-12-04T15:01:37,057 INFO [RS:0;d214edae7bd2:36649 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T15:01:37,057 INFO [RS:0;d214edae7bd2:36649 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d214edae7bd2:36649. 2024-12-04T15:01:37,057 DEBUG [RS:0;d214edae7bd2:36649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T15:01:37,057 DEBUG [RS:0;d214edae7bd2:36649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:37,057 INFO [RS:0;d214edae7bd2:36649 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T15:01:37,057 INFO [RS:0;d214edae7bd2:36649 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T15:01:37,057 INFO [RS:0;d214edae7bd2:36649 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T15:01:37,058 INFO [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T15:01:37,059 INFO [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T15:01:37,059 DEBUG [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-04T15:01:37,059 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T15:01:37,059 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T15:01:37,059 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T15:01:37,059 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T15:01:37,059 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T15:01:37,059 DEBUG [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T15:01:37,059 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-04T15:01:37,079 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/.tmp/info/f49f8c80780a4796bd3ab938ab82c105 is 95, key is TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9./info:/1733324337521/DeleteFamily/seqid=0 2024-12-04T15:01:37,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742492_1668 (size=6514) 2024-12-04T15:01:37,139 INFO [regionserver/d214edae7bd2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T15:01:37,259 DEBUG [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T15:01:37,286 INFO [regionserver/d214edae7bd2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-04T15:01:37,286 INFO [regionserver/d214edae7bd2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-04T15:01:37,459 DEBUG [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T15:01:37,482 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/.tmp/info/f49f8c80780a4796bd3ab938ab82c105 2024-12-04T15:01:37,503 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/.tmp/ns/be6a8157d48e437fac7a02356af819bf is 93, key is TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9./ns:/1733324337487/DeleteFamily/seqid=0 2024-12-04T15:01:37,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742493_1669 (size=5906) 2024-12-04T15:01:37,660 DEBUG [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T15:01:37,860 DEBUG [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T15:01:37,906 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/.tmp/ns/be6a8157d48e437fac7a02356af819bf 2024-12-04T15:01:37,921 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/.tmp/rep_barrier/d6734ec5858d4cf480d97a253fafd23b is 102, key is TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9./rep_barrier:/1733324337487/DeleteFamily/seqid=0 2024-12-04T15:01:37,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742494_1670 (size=6025) 2024-12-04T15:01:38,060 DEBUG [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T15:01:38,260 DEBUG [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T15:01:38,325 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/.tmp/rep_barrier/d6734ec5858d4cf480d97a253fafd23b 2024-12-04T15:01:38,343 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/.tmp/table/50859746bad544b8a977f40c738e0ef8 is 96, key is TestAcidGuarantees,,1733324308805.7fc4055584695509eee4923dfb5273a9./table:/1733324337487/DeleteFamily/seqid=0 2024-12-04T15:01:38,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742495_1671 (size=5936) 2024-12-04T15:01:38,461 DEBUG [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T15:01:38,661 DEBUG [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T15:01:38,747 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/.tmp/table/50859746bad544b8a977f40c738e0ef8 2024-12-04T15:01:38,750 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/.tmp/info/f49f8c80780a4796bd3ab938ab82c105 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/info/f49f8c80780a4796bd3ab938ab82c105 2024-12-04T15:01:38,753 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/info/f49f8c80780a4796bd3ab938ab82c105, entries=12, sequenceid=90, filesize=6.4 K 2024-12-04T15:01:38,754 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/.tmp/ns/be6a8157d48e437fac7a02356af819bf as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/ns/be6a8157d48e437fac7a02356af819bf 2024-12-04T15:01:38,756 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/ns/be6a8157d48e437fac7a02356af819bf, entries=8, sequenceid=90, filesize=5.8 K 2024-12-04T15:01:38,757 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/.tmp/rep_barrier/d6734ec5858d4cf480d97a253fafd23b as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/rep_barrier/d6734ec5858d4cf480d97a253fafd23b 2024-12-04T15:01:38,759 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/rep_barrier/d6734ec5858d4cf480d97a253fafd23b, entries=6, sequenceid=90, filesize=5.9 K 2024-12-04T15:01:38,760 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/.tmp/table/50859746bad544b8a977f40c738e0ef8 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/table/50859746bad544b8a977f40c738e0ef8 2024-12-04T15:01:38,762 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/table/50859746bad544b8a977f40c738e0ef8, entries=7, sequenceid=90, filesize=5.8 K 2024-12-04T15:01:38,762 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1703ms, sequenceid=90, compaction requested=false 2024-12-04T15:01:38,766 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-04T15:01:38,766 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T15:01:38,766 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T15:01:38,766 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733324497059Running coprocessor pre-close hooks at 1733324497059Disabling compacts and flushes for region at 1733324497059Disabling writes for close at 1733324497059Obtaining lock to block concurrent updates at 1733324497059Preparing flush snapshotting stores in 1588230740 at 1733324497059Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733324497060 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733324497060Flushing 1588230740/info: creating writer at 1733324497060Flushing 1588230740/info: appending metadata at 1733324497078 (+18 ms)Flushing 1588230740/info: closing flushed file at 1733324497078Flushing 1588230740/ns: creating writer at 1733324497486 (+408 ms)Flushing 1588230740/ns: appending metadata at 1733324497502 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733324497502Flushing 1588230740/rep_barrier: creating writer at 1733324497909 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733324497921 (+12 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733324497921Flushing 1588230740/table: creating writer at 1733324498328 (+407 ms)Flushing 1588230740/table: appending metadata at 1733324498343 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733324498343Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ae647e5: reopening flushed file at 1733324498750 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a6990de: reopening flushed file at 1733324498753 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28394e54: reopening flushed file at 1733324498757 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23ab8e9d: reopening flushed file at 1733324498759 (+2 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1703ms, sequenceid=90, compaction requested=false at 1733324498762 (+3 ms)Writing region close event to WAL at 1733324498764 (+2 ms)Running coprocessor post-close hooks at 1733324498766 (+2 ms)Closed at 1733324498766 2024-12-04T15:01:38,767 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T15:01:38,861 INFO [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(976): stopping server d214edae7bd2,36649,1733324304801; all regions closed. 2024-12-04T15:01:38,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741834_1010 (size=25344) 2024-12-04T15:01:38,875 DEBUG [RS:0;d214edae7bd2:36649 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/oldWALs 2024-12-04T15:01:38,875 INFO [RS:0;d214edae7bd2:36649 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d214edae7bd2%2C36649%2C1733324304801.meta:.meta(num 1733324308390) 2024-12-04T15:01:38,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741832_1008 (size=22288764) 2024-12-04T15:01:38,880 DEBUG [RS:0;d214edae7bd2:36649 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/oldWALs 2024-12-04T15:01:38,880 INFO [RS:0;d214edae7bd2:36649 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d214edae7bd2%2C36649%2C1733324304801:(num 1733324307444) 2024-12-04T15:01:38,880 DEBUG [RS:0;d214edae7bd2:36649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:01:38,880 INFO [RS:0;d214edae7bd2:36649 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T15:01:38,881 INFO [RS:0;d214edae7bd2:36649 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T15:01:38,881 INFO [RS:0;d214edae7bd2:36649 {}] hbase.ChoreService(370): Chore service for: regionserver/d214edae7bd2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T15:01:38,881 INFO [RS:0;d214edae7bd2:36649 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T15:01:38,881 INFO [regionserver/d214edae7bd2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T15:01:38,882 INFO [RS:0;d214edae7bd2:36649 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36649 2024-12-04T15:01:38,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36649-0x101a2238e4c0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d214edae7bd2,36649,1733324304801 2024-12-04T15:01:38,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T15:01:38,942 INFO [RS:0;d214edae7bd2:36649 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T15:01:38,951 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d214edae7bd2,36649,1733324304801] 2024-12-04T15:01:38,959 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d214edae7bd2,36649,1733324304801 already deleted, retry=false 2024-12-04T15:01:38,959 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d214edae7bd2,36649,1733324304801 expired; onlineServers=0 2024-12-04T15:01:38,960 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd214edae7bd2,39449,1733324304115' ***** 2024-12-04T15:01:38,960 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T15:01:38,960 INFO [M:0;d214edae7bd2:39449 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T15:01:38,961 INFO [M:0;d214edae7bd2:39449 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T15:01:38,961 DEBUG [M:0;d214edae7bd2:39449 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T15:01:38,961 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T15:01:38,961 DEBUG [M:0;d214edae7bd2:39449 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T15:01:38,961 DEBUG [master/d214edae7bd2:0:becomeActiveMaster-HFileCleaner.large.0-1733324307087 {}] cleaner.HFileCleaner(306): Exit Thread[master/d214edae7bd2:0:becomeActiveMaster-HFileCleaner.large.0-1733324307087,5,FailOnTimeoutGroup] 2024-12-04T15:01:38,961 DEBUG [master/d214edae7bd2:0:becomeActiveMaster-HFileCleaner.small.0-1733324307091 {}] cleaner.HFileCleaner(306): Exit Thread[master/d214edae7bd2:0:becomeActiveMaster-HFileCleaner.small.0-1733324307091,5,FailOnTimeoutGroup] 2024-12-04T15:01:38,962 INFO [M:0;d214edae7bd2:39449 {}] hbase.ChoreService(370): Chore service for: master/d214edae7bd2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T15:01:38,962 INFO [M:0;d214edae7bd2:39449 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T15:01:38,963 DEBUG [M:0;d214edae7bd2:39449 {}] master.HMaster(1795): Stopping service threads 2024-12-04T15:01:38,963 INFO [M:0;d214edae7bd2:39449 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T15:01:38,963 INFO [M:0;d214edae7bd2:39449 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T15:01:38,963 ERROR [M:0;d214edae7bd2:39449 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (879580466) connection to localhost/127.0.0.1:39975 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:39975,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-04T15:01:38,964 INFO [M:0;d214edae7bd2:39449 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T15:01:38,965 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T15:01:38,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T15:01:38,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T15:01:38,967 DEBUG [M:0;d214edae7bd2:39449 {}] zookeeper.ZKUtil(347): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T15:01:38,967 WARN [M:0;d214edae7bd2:39449 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T15:01:38,968 INFO [M:0;d214edae7bd2:39449 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/.lastflushedseqids 2024-12-04T15:01:38,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742496_1672 (size=72) 2024-12-04T15:01:39,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36649-0x101a2238e4c0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T15:01:39,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36649-0x101a2238e4c0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T15:01:39,052 INFO [RS:0;d214edae7bd2:36649 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T15:01:39,052 INFO [RS:0;d214edae7bd2:36649 {}] regionserver.HRegionServer(1031): Exiting; stopping=d214edae7bd2,36649,1733324304801; zookeeper connection closed. 2024-12-04T15:01:39,053 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7378beec {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7378beec 2024-12-04T15:01:39,053 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T15:01:39,379 INFO [M:0;d214edae7bd2:39449 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T15:01:39,380 INFO [M:0;d214edae7bd2:39449 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T15:01:39,380 DEBUG [M:0;d214edae7bd2:39449 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T15:01:39,380 INFO [M:0;d214edae7bd2:39449 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T15:01:39,380 DEBUG [M:0;d214edae7bd2:39449 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T15:01:39,380 DEBUG [M:0;d214edae7bd2:39449 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T15:01:39,381 DEBUG [M:0;d214edae7bd2:39449 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T15:01:39,381 INFO [M:0;d214edae7bd2:39449 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=777.32 KB heapSize=952.65 KB 2024-12-04T15:01:39,401 DEBUG [M:0;d214edae7bd2:39449 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e9b2169cbbcf4660957a0f23d000ca4d is 82, key is hbase:meta,,1/info:regioninfo/1733324308468/Put/seqid=0 2024-12-04T15:01:39,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742497_1673 (size=5672) 2024-12-04T15:01:39,806 INFO [M:0;d214edae7bd2:39449 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2166 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e9b2169cbbcf4660957a0f23d000ca4d 2024-12-04T15:01:39,833 DEBUG [M:0;d214edae7bd2:39449 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/13cfe6c0b9ab4603b6ac35247bcda94c is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x8D/proc:d/1733324463939/Put/seqid=0 2024-12-04T15:01:39,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742498_1674 (size=45351) 2024-12-04T15:01:40,237 INFO [M:0;d214edae7bd2:39449 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=776.77 KB at sequenceid=2166 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/13cfe6c0b9ab4603b6ac35247bcda94c 2024-12-04T15:01:40,246 INFO [M:0;d214edae7bd2:39449 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 13cfe6c0b9ab4603b6ac35247bcda94c 2024-12-04T15:01:40,260 DEBUG [M:0;d214edae7bd2:39449 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4809f3318f9f4bd7bd16cbfa06149bf7 is 69, key is d214edae7bd2,36649,1733324304801/rs:state/1733324307217/Put/seqid=0 2024-12-04T15:01:40,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073742499_1675 (size=5156) 2024-12-04T15:01:40,663 INFO [M:0;d214edae7bd2:39449 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2166 (bloomFilter=true), to=hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4809f3318f9f4bd7bd16cbfa06149bf7 2024-12-04T15:01:40,667 DEBUG [M:0;d214edae7bd2:39449 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e9b2169cbbcf4660957a0f23d000ca4d as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e9b2169cbbcf4660957a0f23d000ca4d 2024-12-04T15:01:40,670 INFO [M:0;d214edae7bd2:39449 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e9b2169cbbcf4660957a0f23d000ca4d, entries=8, sequenceid=2166, filesize=5.5 K 2024-12-04T15:01:40,671 DEBUG [M:0;d214edae7bd2:39449 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/13cfe6c0b9ab4603b6ac35247bcda94c as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/13cfe6c0b9ab4603b6ac35247bcda94c 2024-12-04T15:01:40,674 INFO [M:0;d214edae7bd2:39449 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 13cfe6c0b9ab4603b6ac35247bcda94c 2024-12-04T15:01:40,674 INFO [M:0;d214edae7bd2:39449 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/13cfe6c0b9ab4603b6ac35247bcda94c, entries=170, sequenceid=2166, filesize=44.3 K 2024-12-04T15:01:40,675 DEBUG [M:0;d214edae7bd2:39449 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4809f3318f9f4bd7bd16cbfa06149bf7 as hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4809f3318f9f4bd7bd16cbfa06149bf7 2024-12-04T15:01:40,677 INFO [M:0;d214edae7bd2:39449 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39975/user/jenkins/test-data/d276a895-b0cd-bda7-a215-d3d9f6c359ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4809f3318f9f4bd7bd16cbfa06149bf7, entries=1, sequenceid=2166, filesize=5.0 K 2024-12-04T15:01:40,678 INFO [M:0;d214edae7bd2:39449 {}] regionserver.HRegion(3140): Finished flush of dataSize ~777.32 KB/795978, heapSize ~952.35 KB/975208, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1297ms, sequenceid=2166, compaction requested=false 2024-12-04T15:01:40,679 INFO [M:0;d214edae7bd2:39449 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T15:01:40,680 DEBUG [M:0;d214edae7bd2:39449 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733324499380Disabling compacts and flushes for region at 1733324499380Disabling writes for close at 1733324499381 (+1 ms)Obtaining lock to block concurrent updates at 1733324499381Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733324499381Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=795978, getHeapSize=975448, getOffHeapSize=0, getCellsCount=2361 at 1733324499381Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733324499383 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733324499383Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733324499401 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733324499401Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733324499817 (+416 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733324499832 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733324499832Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733324500246 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733324500259 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733324500259Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20dc81ff: reopening flushed file at 1733324500666 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f299e6: reopening flushed file at 1733324500671 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6be95573: reopening flushed file at 1733324500675 (+4 ms)Finished flush of dataSize ~777.32 KB/795978, heapSize ~952.35 KB/975208, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1297ms, sequenceid=2166, compaction requested=false at 1733324500678 (+3 ms)Writing region close event to WAL at 1733324500679 (+1 ms)Closed at 1733324500679 2024-12-04T15:01:40,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44941 is added to blk_1073741830_1006 (size=937967) 2024-12-04T15:01:40,682 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T15:01:40,682 INFO [M:0;d214edae7bd2:39449 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T15:01:40,682 INFO [M:0;d214edae7bd2:39449 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39449 2024-12-04T15:01:40,683 INFO [M:0;d214edae7bd2:39449 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T15:01:40,792 INFO [M:0;d214edae7bd2:39449 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T15:01:40,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T15:01:40,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39449-0x101a2238e4c0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T15:01:40,802 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@292d50c4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T15:01:40,805 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T15:01:40,805 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T15:01:40,805 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T15:01:40,805 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/hadoop.log.dir/,STOPPED} 2024-12-04T15:01:40,807 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T15:01:40,807 WARN [BP-1917135717-172.17.0.2-1733324300869 heartbeating to localhost/127.0.0.1:39975 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T15:01:40,807 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T15:01:40,807 WARN [BP-1917135717-172.17.0.2-1733324300869 heartbeating to localhost/127.0.0.1:39975 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1917135717-172.17.0.2-1733324300869 (Datanode Uuid dc1a60eb-c1c2-4629-8cb3-41f5de55f15a) service to localhost/127.0.0.1:39975 2024-12-04T15:01:40,809 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/cluster_5448908e-37df-abd5-da05-dbe928ab313c/data/data1/current/BP-1917135717-172.17.0.2-1733324300869 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T15:01:40,809 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/cluster_5448908e-37df-abd5-da05-dbe928ab313c/data/data2/current/BP-1917135717-172.17.0.2-1733324300869 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T15:01:40,810 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T15:01:40,815 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2341a969{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T15:01:40,816 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T15:01:40,816 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T15:01:40,816 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T15:01:40,816 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3dd7400d-d677-46e7-38bd-f8803444497b/hadoop.log.dir/,STOPPED} 2024-12-04T15:01:40,830 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T15:01:40,945 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down